vllm.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "R9Ui_WtrQ7Je"
      ],
      "toc_visible": true,
      "gpuType": "T4",
      "authorship_tag": "ABX9TyM6IuzWDXKaYl+/YSCXrtFS",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/uppercaveman/vllm-learn/blob/main/vllm.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 检查Python环境"
      ],
      "metadata": {
        "id": "JYvcHFq0D9tm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip3 --version"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wM7ttGGfEAwZ",
        "outputId": "f8c09f62-453c-4383-90da-90727fcf9135"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "pip 23.1.2 from /usr/local/lib/python3.10/dist-packages/pip (python 3.10)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip --version"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Hp3KHVEN4wKg",
        "outputId": "7cf958ea-1468-4a90-8eba-0a1d09d4f802"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "pip 23.1.2 from /usr/local/lib/python3.10/dist-packages/pip (python 3.10)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 查看GPU信息"
      ],
      "metadata": {
        "id": "iGrIbcuHjiCt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!nvidia-smi\n",
        "# nvidia-smi 是 NVIDIA 提供的一个命令行工具，用于监控和管理 NVIDIA GPU 设备"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vAdXbjUKjlYy",
        "outputId": "9a902c62-126d-445c-aefd-ca8771e6d74b"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mon Apr 22 06:46:42 2024       \n",
            "+---------------------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 535.104.05             Driver Version: 535.104.05   CUDA Version: 12.2     |\n",
            "|-----------------------------------------+----------------------+----------------------+\n",
            "| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                                         |                      |               MIG M. |\n",
            "|=========================================+======================+======================|\n",
            "|   0  Tesla T4                       Off | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   62C    P8              11W /  70W |      0MiB / 15360MiB |      0%      Default |\n",
            "|                                         |                      |                  N/A |\n",
            "+-----------------------------------------+----------------------+----------------------+\n",
            "                                                                                         \n",
            "+---------------------------------------------------------------------------------------+\n",
            "| Processes:                                                                            |\n",
            "|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |\n",
            "|        ID   ID                                                             Usage      |\n",
            "|=======================================================================================|\n",
            "|  No running processes found                                                           |\n",
            "+---------------------------------------------------------------------------------------+\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 更多内存信息\n",
        "# memory footprint support libraries/code\n",
        "!ln -sf /opt/bin/nvidia-smi /usr/bin/nvidia-smi\n",
        "!pip install -q gputil\n",
        "\n",
        "import psutil\n",
        "import humanize\n",
        "import os\n",
        "import GPUtil as GPU\n",
        "\n",
        "GPUs = GPU.getGPUs()\n",
        "# XXX: only one GPU on Colab and isn’t guaranteed\n",
        "gpu = GPUs[0]\n",
        "def printm():\n",
        "    process = psutil.Process(os.getpid())\n",
        "    virtual_memory = humanize.naturalsize( psutil.virtual_memory().available )\n",
        "    memory_info = humanize.naturalsize( process.memory_info().rss)\n",
        "    memoryFree =gpu.memoryFree\n",
        "    memoryUsed = gpu.memoryUsed\n",
        "    memoryUtil = gpu.memoryUtil*100\n",
        "    memoryTotal = gpu.memoryTotal\n",
        "\n",
        "    print(f\"Gen RAM Free: {virtual_memory} | Proc size: {memory_info}\" )\n",
        "\n",
        "    print(\"GPU RAM Free: {0:.0f}MB | Used: {1:.0f}MB | Util {2:3.0f}% | Total {3:.0f}MB\".\\\n",
        "          format(memoryFree, memoryUsed, memoryUtil*100, memoryTotal))\n",
        "printm()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZKta-QWSlJK1",
        "outputId": "ad2998f3-a346-4492-b08c-1ff689ac420f"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for gputil (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Gen RAM Free: 12.6 GB | Proc size: 99.1 MB\n",
            "GPU RAM Free: 15101MB | Used: 0MB | Util   0% | Total 15360MB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 查看cuda版本信息\n",
        "!nvcc -V"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ypt29tyzlcNT",
        "outputId": "174cba08-d59d-455e-ad64-754d47d0144f"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "nvcc: NVIDIA (R) Cuda compiler driver\n",
            "Copyright (c) 2005-2023 NVIDIA Corporation\n",
            "Built on Tue_Aug_15_22:02:13_PDT_2023\n",
            "Cuda compilation tools, release 12.2, V12.2.140\n",
            "Build cuda_12.2.r12.2/compiler.33191640_0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 安装Conda（可选）"
      ],
      "metadata": {
        "id": "R9Ui_WtrQ7Je"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# 检查conda\n",
        "!conda --version"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0LiN89pqS3DZ",
        "outputId": "788ec35a-1354-4c90-9f8d-97fc17828e37"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "conda 24.1.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 下载最新版本conda\n",
        "!wget https://repo.anaconda.com/archive/Anaconda3-2024.02-1-Linux-x86_64.sh"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SE7E9uojTdJn",
        "outputId": "969742f6-7fc8-4cd5-87e9-ee6aa9919113"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2024-04-11 02:25:19--  https://repo.anaconda.com/archive/Anaconda3-2024.02-1-Linux-x86_64.sh\n",
            "Resolving repo.anaconda.com (repo.anaconda.com)... 104.16.130.3, 104.16.131.3, 2606:4700::6810:8203, ...\n",
            "Connecting to repo.anaconda.com (repo.anaconda.com)|104.16.130.3|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 1045673900 (997M) [application/octet-stream]\n",
            "Saving to: ‘Anaconda3-2024.02-1-Linux-x86_64.sh’\n",
            "\n",
            "Anaconda3-2024.02-1 100%[===================>] 997.23M   118MB/s    in 6.8s    \n",
            "\n",
            "2024-04-11 02:25:26 (147 MB/s) - ‘Anaconda3-2024.02-1-Linux-x86_64.sh’ saved [1045673900/1045673900]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 授权\n",
        "!chmod a+x /content/Anaconda3-2024.02-1-Linux-x86_64.sh"
      ],
      "metadata": {
        "id": "7U51P8yIUNEN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 安装conda\n",
        "!/content/Anaconda3-2024.02-1-Linux-x86_64.sh -b"
      ],
      "metadata": {
        "id": "GrwSyWlWUPHb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3a121bbc-3f73-472c-878d-b0be6527406b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "PREFIX=/root/anaconda3\n",
            "Unpacking payload ...\n",
            "                                                                                               \n",
            "Installing base environment...\n",
            "\n",
            "\n",
            "Downloading and Extracting Packages:\n",
            "\n",
            "\n",
            "Downloading and Extracting Packages:\n",
            "\n",
            "Preparing transaction: - \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\bdone\n",
            "Executing transaction: - \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \n",
            "\n",
            "    Installed package of scikit-learn can be accelerated using scikit-learn-intelex.\n",
            "    More details are available here: https://intel.github.io/scikit-learn-intelex\n",
            "\n",
            "    For example:\n",
            "\n",
            "        $ conda install scikit-learn-intelex\n",
            "        $ python -m sklearnex my_application.py\n",
            "\n",
            "    \n",
            "\n",
            "\b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\bdone\n",
            "installation finished.\n",
            "WARNING:\n",
            "    You currently have a PYTHONPATH environment variable set. This may cause\n",
            "    unexpected behavior when running the Python interpreter in Anaconda3.\n",
            "    For best results, please verify that your PYTHONPATH only points to\n",
            "    directories of packages that are compatible with the Python interpreter\n",
            "    in Anaconda3: /root/anaconda3\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 配置环境变量\n",
        "%env PATH=\"/root/anaconda3/bin:\"&&echo $PATH\n",
        "!source ~/.bashrc\n",
        "!echo $PATH"
      ],
      "metadata": {
        "id": "1nDpsMONOmPK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6dbf2b1d-d823-400b-ee3b-2bf47f3b68f5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "env: PATH=\"/root/anaconda3/bin:\"&&echo $PATH\n",
            "\"/root/anaconda3/bin:\"&&echo $PATH\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%env PATH=/root/anaconda3/bin:/opt/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/tools/node/bin:/tools/google-cloud-sdk/bin"
      ],
      "metadata": {
        "id": "YeL26omHWupJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "208d6781-90ec-4f73-c84b-0a7b201bea25"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "env: PATH=/root/anaconda3/bin:/opt/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/tools/node/bin:/tools/google-cloud-sdk/bin\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 配置conda空间\n",
        "!conda create -n vllmspace python=3.11 -y\n",
        "!conda activate vllmspace"
      ],
      "metadata": {
        "id": "6ACArOLl987H",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1601f429-af7f-4aca-8e45-c92c586255d8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Channels:\n",
            " - defaults\n",
            "Platform: linux-64\n",
            "Collecting package metadata (repodata.json): - \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\bdone\n",
            "Solving environment: \\ \b\bdone\n",
            "\n",
            "## Package Plan ##\n",
            "\n",
            "  environment location: /root/anaconda3/envs/vllmspace\n",
            "\n",
            "  added / updated specs:\n",
            "    - python=3.11\n",
            "\n",
            "\n",
            "The following packages will be downloaded:\n",
            "\n",
            "    package                    |            build\n",
            "    ---------------------------|-----------------\n",
            "    bzip2-1.0.8                |       h5eee18b_5         262 KB\n",
            "    ca-certificates-2024.3.11  |       h06a4308_0         127 KB\n",
            "    python-3.11.8              |       h955ad1f_0        32.9 MB\n",
            "    tzdata-2024a               |       h04d1e81_0         116 KB\n",
            "    xz-5.4.6                   |       h5eee18b_0         651 KB\n",
            "    ------------------------------------------------------------\n",
            "                                           Total:        34.0 MB\n",
            "\n",
            "The following NEW packages will be INSTALLED:\n",
            "\n",
            "  _libgcc_mutex      pkgs/main/linux-64::_libgcc_mutex-0.1-main \n",
            "  _openmp_mutex      pkgs/main/linux-64::_openmp_mutex-5.1-1_gnu \n",
            "  bzip2              pkgs/main/linux-64::bzip2-1.0.8-h5eee18b_5 \n",
            "  ca-certificates    pkgs/main/linux-64::ca-certificates-2024.3.11-h06a4308_0 \n",
            "  ld_impl_linux-64   pkgs/main/linux-64::ld_impl_linux-64-2.38-h1181459_1 \n",
            "  libffi             pkgs/main/linux-64::libffi-3.4.4-h6a678d5_0 \n",
            "  libgcc-ng          pkgs/main/linux-64::libgcc-ng-11.2.0-h1234567_1 \n",
            "  libgomp            pkgs/main/linux-64::libgomp-11.2.0-h1234567_1 \n",
            "  libstdcxx-ng       pkgs/main/linux-64::libstdcxx-ng-11.2.0-h1234567_1 \n",
            "  libuuid            pkgs/main/linux-64::libuuid-1.41.5-h5eee18b_0 \n",
            "  ncurses            pkgs/main/linux-64::ncurses-6.4-h6a678d5_0 \n",
            "  openssl            pkgs/main/linux-64::openssl-3.0.13-h7f8727e_0 \n",
            "  pip                pkgs/main/linux-64::pip-23.3.1-py311h06a4308_0 \n",
            "  python             pkgs/main/linux-64::python-3.11.8-h955ad1f_0 \n",
            "  readline           pkgs/main/linux-64::readline-8.2-h5eee18b_0 \n",
            "  setuptools         pkgs/main/linux-64::setuptools-68.2.2-py311h06a4308_0 \n",
            "  sqlite             pkgs/main/linux-64::sqlite-3.41.2-h5eee18b_0 \n",
            "  tk                 pkgs/main/linux-64::tk-8.6.12-h1ccaba5_0 \n",
            "  tzdata             pkgs/main/noarch::tzdata-2024a-h04d1e81_0 \n",
            "  wheel              pkgs/main/linux-64::wheel-0.41.2-py311h06a4308_0 \n",
            "  xz                 pkgs/main/linux-64::xz-5.4.6-h5eee18b_0 \n",
            "  zlib               pkgs/main/linux-64::zlib-1.2.13-h5eee18b_0 \n",
            "\n",
            "\n",
            "\n",
            "Downloading and Extracting Packages:\n",
            "python-3.11.8        | 32.9 MB   | :   0% 0/1 [00:00<?, ?it/s]\n",
            "xz-5.4.6             | 651 KB    | :   0% 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "\n",
            "bzip2-1.0.8          | 262 KB    | :   0% 0/1 [00:00<?, ?it/s]\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "ca-certificates-2024 | 127 KB    | :   0% 0/1 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "python-3.11.8        | 32.9 MB   | :   0% 0.0004748507903697802/1 [00:00<05:26, 326.72s/it]\n",
            "\n",
            "bzip2-1.0.8          | 262 KB    | :   6% 0.061068247046106826/1 [00:00<00:02,  3.02s/it]\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "ca-certificates-2024 | 127 KB    | :  13% 0.12568368888991172/1 [00:00<00:01,  1.66s/it]\u001b[A\u001b[A\u001b[A\n",
            "xz-5.4.6             | 651 KB    | :   2% 0.024589524238331082/1 [00:00<00:08,  8.70s/it]\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "python-3.11.8        | 32.9 MB   | :  17% 0.1728456876946/1 [00:00<00:01,  1.21s/it]       \n",
            "\n",
            "bzip2-1.0.8          | 262 KB    | : 100% 1.0/1 [00:00<00:00,  3.02s/it]                 \u001b[A\u001b[A\n",
            "\n",
            "\n",
            "python-3.11.8        | 32.9 MB   | :  87% 0.872775752699656/1 [00:00<00:00,  1.58it/s] \n",
            "xz-5.4.6             | 651 KB    | : 100% 1.0/1 [00:00<00:00,  1.63it/s]                 \u001b[A\n",
            "xz-5.4.6             | 651 KB    | : 100% 1.0/1 [00:00<00:00,  1.63it/s]\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "tzdata-2024a         | 116 KB    | : 100% 1.0/1 [00:00<00:00,  1.40it/s]                \u001b[A\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "                                                                        \n",
            "                                                                        \u001b[A\n",
            "\n",
            "                                                                        \u001b[A\u001b[A\n",
            "\n",
            "\n",
            "                                                                        \u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "\n",
            "                                                                        \u001b[A\u001b[A\u001b[A\u001b[A\n",
            "Preparing transaction: / \b\b- \b\bdone\n",
            "Verifying transaction: | \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\bdone\n",
            "Executing transaction: / \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\bdone\n",
            "#\n",
            "# To activate this environment, use\n",
            "#\n",
            "#     $ conda activate vllmspace\n",
            "#\n",
            "# To deactivate an active environment, use\n",
            "#\n",
            "#     $ conda deactivate\n",
            "\n",
            "\n",
            "CondaError: Run 'conda init' before 'conda activate'\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 单独安装，升级为最新版本\n",
        "!pip install anaconda-cloud-auth==0.5.0"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v_P9F5VWqZBB",
        "outputId": "56098d25-d17e-4aef-f457-6ba01caf3451"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting anaconda-cloud-auth==0.5.0\n",
            "  Downloading anaconda_cloud_auth-0.5.0-py3-none-any.whl.metadata (10 kB)\n",
            "Requirement already satisfied: cryptography>=3.4.0 in /root/anaconda3/lib/python3.11/site-packages (from anaconda-cloud-auth==0.5.0) (42.0.2)\n",
            "Requirement already satisfied: keyring in /root/anaconda3/lib/python3.11/site-packages (from anaconda-cloud-auth==0.5.0) (23.13.1)\n",
            "Requirement already satisfied: pkce in /root/anaconda3/lib/python3.11/site-packages (from anaconda-cloud-auth==0.5.0) (1.0.3)\n",
            "Requirement already satisfied: pydantic in /root/anaconda3/lib/python3.11/site-packages (from anaconda-cloud-auth==0.5.0) (1.10.12)\n",
            "Requirement already satisfied: pyjwt in /root/anaconda3/lib/python3.11/site-packages (from anaconda-cloud-auth==0.5.0) (2.4.0)\n",
            "Requirement already satisfied: python-dotenv in /root/anaconda3/lib/python3.11/site-packages (from anaconda-cloud-auth==0.5.0) (0.21.0)\n",
            "Requirement already satisfied: requests in /root/anaconda3/lib/python3.11/site-packages (from anaconda-cloud-auth==0.5.0) (2.31.0)\n",
            "Requirement already satisfied: semver<4 in /root/anaconda3/lib/python3.11/site-packages (from anaconda-cloud-auth==0.5.0) (2.13.0)\n",
            "Requirement already satisfied: cffi>=1.12 in /root/anaconda3/lib/python3.11/site-packages (from cryptography>=3.4.0->anaconda-cloud-auth==0.5.0) (1.16.0)\n",
            "Requirement already satisfied: jaraco.classes in /root/anaconda3/lib/python3.11/site-packages (from keyring->anaconda-cloud-auth==0.5.0) (3.2.1)\n",
            "Requirement already satisfied: importlib-metadata>=4.11.4 in /root/anaconda3/lib/python3.11/site-packages (from keyring->anaconda-cloud-auth==0.5.0) (7.0.1)\n",
            "Requirement already satisfied: SecretStorage>=3.2 in /root/anaconda3/lib/python3.11/site-packages (from keyring->anaconda-cloud-auth==0.5.0) (3.3.1)\n",
            "Requirement already satisfied: jeepney>=0.4.2 in /root/anaconda3/lib/python3.11/site-packages (from keyring->anaconda-cloud-auth==0.5.0) (0.7.1)\n",
            "Requirement already satisfied: typing-extensions>=4.2.0 in /root/anaconda3/lib/python3.11/site-packages (from pydantic->anaconda-cloud-auth==0.5.0) (4.9.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /root/anaconda3/lib/python3.11/site-packages (from requests->anaconda-cloud-auth==0.5.0) (2.0.4)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /root/anaconda3/lib/python3.11/site-packages (from requests->anaconda-cloud-auth==0.5.0) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /root/anaconda3/lib/python3.11/site-packages (from requests->anaconda-cloud-auth==0.5.0) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /root/anaconda3/lib/python3.11/site-packages (from requests->anaconda-cloud-auth==0.5.0) (2024.2.2)\n",
            "Requirement already satisfied: pycparser in /root/anaconda3/lib/python3.11/site-packages (from cffi>=1.12->cryptography>=3.4.0->anaconda-cloud-auth==0.5.0) (2.21)\n",
            "Requirement already satisfied: zipp>=0.5 in /root/anaconda3/lib/python3.11/site-packages (from importlib-metadata>=4.11.4->keyring->anaconda-cloud-auth==0.5.0) (3.17.0)\n",
            "Requirement already satisfied: more-itertools in /root/anaconda3/lib/python3.11/site-packages (from jaraco.classes->keyring->anaconda-cloud-auth==0.5.0) (10.1.0)\n",
            "Downloading anaconda_cloud_auth-0.5.0-py3-none-any.whl (24 kB)\n",
            "Installing collected packages: anaconda-cloud-auth\n",
            "  Attempting uninstall: anaconda-cloud-auth\n",
            "    Found existing installation: anaconda-cloud-auth 0.1.4\n",
            "    Uninstalling anaconda-cloud-auth-0.1.4:\n",
            "      Successfully uninstalled anaconda-cloud-auth-0.1.4\n",
            "Successfully installed anaconda-cloud-auth-0.5.0\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0m"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 安装vllm"
      ],
      "metadata": {
        "id": "k5cYlOcl92XA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# 升级PyTorch，CPU版\n",
        "!pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4WSd6TUV3qUM",
        "outputId": "270471d2-7fa8-493d-b2bf-4803af57795a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://download.pytorch.org/whl/cpu\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (2.2.1+cu121)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.10/dist-packages (0.17.1+cu121)\n",
            "Requirement already satisfied: torchaudio in /usr/local/lib/python3.10/dist-packages (2.2.1+cu121)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch) (3.13.4)\n",
            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch) (4.11.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch) (3.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch) (3.1.3)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch) (2023.6.0)\n",
            "INFO: pip is looking at multiple versions of torch to determine which version is compatible with other requirements. This could take a while.\n",
            "Collecting torch\n",
            "  Downloading https://download.pytorch.org/whl/cpu/torch-2.2.2%2Bcpu-cp310-cp310-linux_x86_64.whl (186.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m186.8/186.8 MB\u001b[0m \u001b[31m6.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from torchvision) (1.25.2)\n",
            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.10/dist-packages (from torchvision) (9.4.0)\n",
            "  Downloading https://download.pytorch.org/whl/cpu/torch-2.2.1%2Bcpu-cp310-cp310-linux_x86_64.whl (186.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m186.8/186.8 MB\u001b[0m \u001b[31m6.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch) (2.1.5)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch) (1.3.0)\n",
            "Installing collected packages: torch\n",
            "  Attempting uninstall: torch\n",
            "    Found existing installation: torch 2.2.1+cu121\n",
            "    Uninstalling torch-2.2.1+cu121:\n",
            "      Successfully uninstalled torch-2.2.1+cu121\n",
            "Successfully installed torch-2.2.1+cpu\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 安装vllm\n",
        "!pip3 install vllm"
      ],
      "metadata": {
        "id": "4yNkSDXFXYYG",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d2a5fa32-c5d7-4b6c-b8c0-d9b4f3063cfc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting vllm\n",
            "  Downloading vllm-0.4.0.post1-cp310-cp310-manylinux1_x86_64.whl (97.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m97.6/97.6 MB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: cmake>=3.21 in /usr/local/lib/python3.10/dist-packages (from vllm) (3.27.9)\n",
            "Collecting ninja (from vllm)\n",
            "  Downloading ninja-1.11.1.1-py2.py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl (307 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.2/307.2 kB\u001b[0m \u001b[31m27.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from vllm) (5.9.5)\n",
            "Collecting ray>=2.9 (from vllm)\n",
            "  Downloading ray-2.10.0-cp310-cp310-manylinux2014_x86_64.whl (65.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m65.1/65.1 MB\u001b[0m \u001b[31m7.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: sentencepiece in /usr/local/lib/python3.10/dist-packages (from vllm) (0.1.99)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from vllm) (1.25.2)\n",
            "Collecting torch==2.1.2 (from vllm)\n",
            "  Downloading torch-2.1.2-cp310-cp310-manylinux1_x86_64.whl (670.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m670.2/670.2 MB\u001b[0m \u001b[31m703.8 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from vllm) (2.31.0)\n",
            "Requirement already satisfied: py-cpuinfo in /usr/local/lib/python3.10/dist-packages (from vllm) (9.0.0)\n",
            "Collecting transformers>=4.39.1 (from vllm)\n",
            "  Downloading transformers-4.39.3-py3-none-any.whl (8.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.8/8.8 MB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting xformers==0.0.23.post1 (from vllm)\n",
            "  Downloading xformers-0.0.23.post1-cp310-cp310-manylinux2014_x86_64.whl (213.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m213.0/213.0 MB\u001b[0m \u001b[31m1.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting fastapi (from vllm)\n",
            "  Downloading fastapi-0.110.1-py3-none-any.whl (91 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m91.9/91.9 kB\u001b[0m \u001b[31m1.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting uvicorn[standard] (from vllm)\n",
            "  Downloading uvicorn-0.29.0-py3-none-any.whl (60 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.8/60.8 kB\u001b[0m \u001b[31m1.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pydantic>=2.0 in /usr/local/lib/python3.10/dist-packages (from vllm) (2.6.4)\n",
            "Requirement already satisfied: prometheus-client>=0.18.0 in /usr/local/lib/python3.10/dist-packages (from vllm) (0.20.0)\n",
            "Collecting pynvml==11.5.0 (from vllm)\n",
            "  Downloading pynvml-11.5.0-py3-none-any.whl (53 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.1/53.1 kB\u001b[0m \u001b[31m1.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: triton>=2.1.0 in /usr/local/lib/python3.10/dist-packages (from vllm) (2.2.0)\n",
            "Collecting outlines==0.0.34 (from vllm)\n",
            "  Downloading outlines-0.0.34-py3-none-any.whl (76 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.5/76.5 kB\u001b[0m \u001b[31m1.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting tiktoken==0.6.0 (from vllm)\n",
            "  Downloading tiktoken-0.6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m1.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting interegular (from outlines==0.0.34->vllm)\n",
            "  Downloading interegular-0.3.3-py37-none-any.whl (23 kB)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from outlines==0.0.34->vllm) (3.1.3)\n",
            "Collecting lark (from outlines==0.0.34->vllm)\n",
            "  Downloading lark-1.1.9-py3-none-any.whl (111 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m111.7/111.7 kB\u001b[0m \u001b[31m1.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: nest-asyncio in /usr/local/lib/python3.10/dist-packages (from outlines==0.0.34->vllm) (1.6.0)\n",
            "Requirement already satisfied: cloudpickle in /usr/local/lib/python3.10/dist-packages (from outlines==0.0.34->vllm) (2.2.1)\n",
            "Collecting diskcache (from outlines==0.0.34->vllm)\n",
            "  Downloading diskcache-5.6.3-py3-none-any.whl (45 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.5/45.5 kB\u001b[0m \u001b[31m1.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from outlines==0.0.34->vllm) (1.11.4)\n",
            "Requirement already satisfied: numba in /usr/local/lib/python3.10/dist-packages (from outlines==0.0.34->vllm) (0.58.1)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from outlines==0.0.34->vllm) (1.4.0)\n",
            "Requirement already satisfied: referencing in /usr/local/lib/python3.10/dist-packages (from outlines==0.0.34->vllm) (0.34.0)\n",
            "Requirement already satisfied: jsonschema in /usr/local/lib/python3.10/dist-packages (from outlines==0.0.34->vllm) (4.19.2)\n",
            "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken==0.6.0->vllm) (2023.12.25)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch==2.1.2->vllm) (3.13.4)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch==2.1.2->vllm) (4.11.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch==2.1.2->vllm) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch==2.1.2->vllm) (3.3)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch==2.1.2->vllm) (2023.6.0)\n",
            "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch==2.1.2->vllm)\n",
            "  Using cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
            "Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch==2.1.2->vllm)\n",
            "  Using cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
            "Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch==2.1.2->vllm)\n",
            "  Using cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
            "Collecting nvidia-cudnn-cu12==8.9.2.26 (from torch==2.1.2->vllm)\n",
            "  Using cached nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n",
            "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch==2.1.2->vllm)\n",
            "  Using cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
            "Collecting nvidia-cufft-cu12==11.0.2.54 (from torch==2.1.2->vllm)\n",
            "  Using cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
            "Collecting nvidia-curand-cu12==10.3.2.106 (from torch==2.1.2->vllm)\n",
            "  Using cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
            "Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch==2.1.2->vllm)\n",
            "  Using cached nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
            "Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch==2.1.2->vllm)\n",
            "  Using cached nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
            "Collecting nvidia-nccl-cu12==2.18.1 (from torch==2.1.2->vllm)\n",
            "  Downloading nvidia_nccl_cu12-2.18.1-py3-none-manylinux1_x86_64.whl (209.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m209.8/209.8 MB\u001b[0m \u001b[31m1.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-nvtx-cu12==12.1.105 (from torch==2.1.2->vllm)\n",
            "  Using cached nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
            "Collecting triton>=2.1.0 (from vllm)\n",
            "  Downloading triton-2.1.0-0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (89.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m89.2/89.2 MB\u001b[0m \u001b[31m1.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h\u001b[33mWARNING: Retrying (Retry(total=4, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ProtocolError('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))': /simple/nvidia-nvjitlink-cu12/\u001b[0m\u001b[33m\n",
            "\u001b[0mCollecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch==2.1.2->vllm)\n",
            "  Using cached nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl (21.1 MB)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2.0->vllm) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.16.3 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2.0->vllm) (2.16.3)\n",
            "Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.10/dist-packages (from ray>=2.9->vllm) (8.1.7)\n",
            "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from ray>=2.9->vllm) (1.0.8)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from ray>=2.9->vllm) (24.0)\n",
            "Requirement already satisfied: protobuf!=3.19.5,>=3.15.3 in /usr/local/lib/python3.10/dist-packages (from ray>=2.9->vllm) (3.20.3)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from ray>=2.9->vllm) (6.0.1)\n",
            "Requirement already satisfied: aiosignal in /usr/local/lib/python3.10/dist-packages (from ray>=2.9->vllm) (1.3.1)\n",
            "Requirement already satisfied: frozenlist in /usr/local/lib/python3.10/dist-packages (from ray>=2.9->vllm) (1.4.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->vllm) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->vllm) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->vllm) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->vllm) (2024.2.2)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.39.1->vllm) (0.20.3)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.39.1->vllm) (0.15.2)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.39.1->vllm) (0.4.2)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.39.1->vllm) (4.66.2)\n",
            "Collecting starlette<0.38.0,>=0.37.2 (from fastapi->vllm)\n",
            "  Downloading starlette-0.37.2-py3-none-any.whl (71 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.9/71.9 kB\u001b[0m \u001b[31m1.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting h11>=0.8 (from uvicorn[standard]->vllm)\n",
            "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m974.3 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting httptools>=0.5.0 (from uvicorn[standard]->vllm)\n",
            "  Downloading httptools-0.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (341 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m341.4/341.4 kB\u001b[0m \u001b[31m1.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting python-dotenv>=0.13 (from uvicorn[standard]->vllm)\n",
            "  Downloading python_dotenv-1.0.1-py3-none-any.whl (19 kB)\n",
            "Collecting uvloop!=0.15.0,!=0.15.1,>=0.14.0 (from uvicorn[standard]->vllm)\n",
            "  Downloading uvloop-0.19.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m1.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting watchfiles>=0.13 (from uvicorn[standard]->vllm)\n",
            "  Downloading watchfiles-0.21.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m1.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting websockets>=10.4 (from uvicorn[standard]->vllm)\n",
            "  Downloading websockets-12.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (130 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m130.2/130.2 kB\u001b[0m \u001b[31m1.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: anyio<5,>=3.4.0 in /usr/local/lib/python3.10/dist-packages (from starlette<0.38.0,>=0.37.2->fastapi->vllm) (3.7.1)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->outlines==0.0.34->vllm) (2.1.5)\n",
            "Requirement already satisfied: attrs>=22.2.0 in /usr/local/lib/python3.10/dist-packages (from jsonschema->outlines==0.0.34->vllm) (23.2.0)\n",
            "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.10/dist-packages (from jsonschema->outlines==0.0.34->vllm) (2023.12.1)\n",
            "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from jsonschema->outlines==0.0.34->vllm) (0.18.0)\n",
            "Requirement already satisfied: llvmlite<0.42,>=0.41.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba->outlines==0.0.34->vllm) (0.41.1)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch==2.1.2->vllm) (1.3.0)\n",
            "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.4.0->starlette<0.38.0,>=0.37.2->fastapi->vllm) (1.3.1)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.4.0->starlette<0.38.0,>=0.37.2->fastapi->vllm) (1.2.0)\n",
            "Installing collected packages: ninja, websockets, uvloop, triton, python-dotenv, pynvml, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, lark, interegular, httptools, h11, diskcache, watchfiles, uvicorn, tiktoken, starlette, nvidia-cusparse-cu12, nvidia-cudnn-cu12, nvidia-cusolver-cu12, fastapi, transformers, torch, ray, xformers, outlines, vllm\n",
            "  Attempting uninstall: triton\n",
            "    Found existing installation: triton 2.2.0\n",
            "    Uninstalling triton-2.2.0:\n",
            "      Successfully uninstalled triton-2.2.0\n",
            "  Attempting uninstall: transformers\n",
            "    Found existing installation: transformers 4.38.2\n",
            "    Uninstalling transformers-4.38.2:\n",
            "      Successfully uninstalled transformers-4.38.2\n",
            "  Attempting uninstall: torch\n",
            "    Found existing installation: torch 2.2.1+cpu\n",
            "    Uninstalling torch-2.2.1+cpu:\n",
            "      Successfully uninstalled torch-2.2.1+cpu\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "torchaudio 2.2.1+cu121 requires torch==2.2.1, but you have torch 2.1.2 which is incompatible.\n",
            "torchtext 0.17.1 requires torch==2.2.1, but you have torch 2.1.2 which is incompatible.\n",
            "torchvision 0.17.1+cu121 requires torch==2.2.1, but you have torch 2.1.2 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed diskcache-5.6.3 fastapi-0.110.1 h11-0.14.0 httptools-0.6.1 interegular-0.3.3 lark-1.1.9 ninja-1.11.1.1 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.18.1 nvidia-nvjitlink-cu12-12.4.127 nvidia-nvtx-cu12-12.1.105 outlines-0.0.34 pynvml-11.5.0 python-dotenv-1.0.1 ray-2.10.0 starlette-0.37.2 tiktoken-0.6.0 torch-2.1.2 transformers-4.39.3 triton-2.1.0 uvicorn-0.29.0 uvloop-0.19.0 vllm-0.4.0.post1 watchfiles-0.21.0 websockets-12.0 xformers-0.0.23.post1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 下载 chatglm3-6b 模型"
      ],
      "metadata": {
        "id": "ekupwkmtCVZf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# 从 HuggingFace 下载 chatglm3-6b 模型\n",
        "!git clone https://huggingface.co/THUDM/chatglm3-6b"
      ],
      "metadata": {
        "id": "hb0OMihdCWPr",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "582f5832-53ce-4f25-d453-a94cd93274f6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'chatglm3-6b'...\n",
            "remote: Enumerating objects: 128, done.\u001b[K\n",
            "remote: Counting objects: 100% (125/125), done.\u001b[K\n",
            "remote: Compressing objects: 100% (124/124), done.\u001b[K\n",
            "remote: Total 128 (delta 62), reused 0 (delta 0), pack-reused 3\u001b[K\n",
            "Receiving objects: 100% (128/128), 51.37 KiB | 17.12 MiB/s, done.\n",
            "Resolving deltas: 100% (62/62), done.\n",
            "Filtering content: 100% (15/15), 23.26 GiB | 39.14 MiB/s, done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 测试"
      ],
      "metadata": {
        "id": "PwZ-nbDAwDom"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import vllm\n",
        "# 导入LLM, SamplingParams\n",
        "from vllm import LLM, SamplingParams"
      ],
      "metadata": {
        "id": "CrZsYJiswHR7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 定义输入提示列表和生成的采样参数\n",
        "prompts = [\n",
        "    \"你好，我的名字是\",\n",
        "    \"中国首都是\",\n",
        "    \"北京水库是\",\n",
        "    \"AI的未来是\",\n",
        "]\n",
        "sampling_params = SamplingParams(temperature=0.8, top_p=0.95)"
      ],
      "metadata": {
        "id": "VWtSVYln6JMu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 初始化 vLLM 的引擎\n",
        "# 远程应用模型 llm = LLM(model=\"THUDM/chatglm3-6b\",trust_remote_code=True)\n",
        "# 本地应用模型\n",
        "llm = LLM(model=\"THUDM/chatglm3-6b\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "referenced_widgets": [
            "f7b137d45efd4f7ea99cfe5db3a33eae",
            "171f912fb7984b2d9b9b8251b1d7ab39",
            "04f72c6b14a84756979c35f4e26057e7",
            "2f300d0fe2864613a89b42d71834fea8",
            "9ae925aae3df497da865a35bcb0f4833"
          ]
        },
        "id": "P4h_M0tW7j1z",
        "outputId": "49749ea9-0a5d-4ec6-d96a-5b98060a6dd5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "configuration_chatglm.py:   0%|          | 0.00/2.33k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f7b137d45efd4f7ea99cfe5db3a33eae"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "A new version of the following files was downloaded from https://huggingface.co/THUDM/chatglm3-6b:\n",
            "- configuration_chatglm.py\n",
            ". Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO 04-12 10:04:51 llm_engine.py:74] Initializing an LLM engine (v0.4.0.post1) with config: model='THUDM/chatglm3-6b', tokenizer='THUDM/chatglm3-6b', tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.float16, max_seq_len=8192, download_dir=None, load_format=auto, tensor_parallel_size=1, disable_custom_all_reduce=True, quantization=None, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, seed=0)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/1.40k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "171f912fb7984b2d9b9b8251b1d7ab39"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenization_chatglm.py:   0%|          | 0.00/13.0k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "04f72c6b14a84756979c35f4e26057e7"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "A new version of the following files was downloaded from https://huggingface.co/THUDM/chatglm3-6b:\n",
            "- tokenization_chatglm.py\n",
            ". Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.model:   0%|          | 0.00/1.02M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2f300d0fe2864613a89b42d71834fea8"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "special_tokens_map.json:   0%|          | 0.00/3.00 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9ae925aae3df497da865a35bcb0f4833"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:transformers_modules.THUDM.chatglm3-6b.103caa40027ebfd8450289ca2f278eac4ff26405.tokenization_chatglm:Setting eos_token is not supported, use the default one.\n",
            "WARNING:transformers_modules.THUDM.chatglm3-6b.103caa40027ebfd8450289ca2f278eac4ff26405.tokenization_chatglm:Setting pad_token is not supported, use the default one.\n",
            "WARNING:transformers_modules.THUDM.chatglm3-6b.103caa40027ebfd8450289ca2f278eac4ff26405.tokenization_chatglm:Setting unk_token is not supported, use the default one.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "WARNING 04-12 10:04:52 tokenizer.py:104] Using a slow tokenizer. This might cause a significant slowdown. Consider using a fast tokenizer instead.\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "ImportError",
          "evalue": "libcuda.so.1: cannot open shared object file: No such file or directory",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-6-f42d202c5bea>\u001b[0m in \u001b[0;36m<cell line: 4>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;31m# 远程应用模型 llm = LLM(model=\"THUDM/chatglm3-6b\",trust_remote_code=True)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;31m# 本地应用模型\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mllm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mLLM\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"THUDM/chatglm3-6b\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtrust_remote_code\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/vllm/entrypoints/llm.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, model, tokenizer, tokenizer_mode, trust_remote_code, tensor_parallel_size, dtype, quantization, revision, tokenizer_revision, seed, gpu_memory_utilization, swap_space, enforce_eager, max_context_len_to_capture, disable_custom_all_reduce, **kwargs)\u001b[0m\n\u001b[1;32m    110\u001b[0m             \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    111\u001b[0m         )\n\u001b[0;32m--> 112\u001b[0;31m         self.llm_engine = LLMEngine.from_engine_args(\n\u001b[0m\u001b[1;32m    113\u001b[0m             engine_args, usage_context=UsageContext.LLM_CLASS)\n\u001b[1;32m    114\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequest_counter\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCounter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/vllm/engine/llm_engine.py\u001b[0m in \u001b[0;36mfrom_engine_args\u001b[0;34m(cls, engine_args, usage_context)\u001b[0m\n\u001b[1;32m    194\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    195\u001b[0m         \u001b[0;31m# Create the LLM engine.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 196\u001b[0;31m         engine = cls(\n\u001b[0m\u001b[1;32m    197\u001b[0m             \u001b[0;34m*\u001b[0m\u001b[0mengine_configs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    198\u001b[0m             \u001b[0mexecutor_class\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mexecutor_class\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/vllm/engine/llm_engine.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, model_config, cache_config, parallel_config, scheduler_config, device_config, lora_config, vision_language_config, executor_class, log_stats, usage_context)\u001b[0m\n\u001b[1;32m    108\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mseq_counter\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCounter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    109\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 110\u001b[0;31m         self.model_executor = executor_class(model_config, cache_config,\n\u001b[0m\u001b[1;32m    111\u001b[0m                                              \u001b[0mparallel_config\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscheduler_config\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    112\u001b[0m                                              \u001b[0mdevice_config\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlora_config\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/vllm/executor/gpu_executor.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, model_config, cache_config, parallel_config, scheduler_config, device_config, lora_config, vision_language_config)\u001b[0m\n\u001b[1;32m     35\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m         \u001b[0;31m# Instantiate the worker and load the model to GPU.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 37\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_init_worker\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     38\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     39\u001b[0m         \u001b[0;31m# Profile the memory usage and initialize the cache.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/vllm/executor/gpu_executor.py\u001b[0m in \u001b[0;36m_init_worker\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     43\u001b[0m         \u001b[0;31m# Lazy import the Worker to avoid importing torch.cuda/xformers\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m         \u001b[0;31m# before CUDA_VISIBLE_DEVICES is set in the Worker\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 45\u001b[0;31m         \u001b[0;32mfrom\u001b[0m \u001b[0mvllm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mworker\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mworker\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mWorker\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     46\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     47\u001b[0m         assert self.parallel_config.world_size == 1, (\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/vllm/worker/worker.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mvllm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msequence\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mSamplerOutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSequenceGroupMetadata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mvllm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mworker\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcache_engine\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mCacheEngine\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mvllm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mworker\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel_runner\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mModelRunner\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/vllm/worker/model_runner.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mvllm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlora\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mworker_manager\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mLRUCacheWorkerLoRAManager\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mvllm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel_executor\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mSamplingMetadata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mvllm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel_executor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel_loader\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mget_model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mvllm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel_executor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparallel_utils\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mcustom_all_reduce\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpynccl_utils\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m from vllm.model_executor.parallel_utils.communication_op import (\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/vllm/model_executor/model_loader.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mvllm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mDeviceConfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mModelConfig\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mvllm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel_executor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mModelRegistry\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mvllm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel_executor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mllava\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mLlavaForConditionalGeneration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m from vllm.model_executor.weight_utils import (get_quant_config,\n\u001b[1;32m     12\u001b[0m                                               initialize_dummy_weights)\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/vllm/model_executor/models/llava.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mvllm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mattention\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mAttentionMetadata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mvllm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mVisionLanguageConfig\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mvllm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel_executor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mactivation\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mget_act_fn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     12\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mvllm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel_executor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mLinearMethodBase\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mvllm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel_executor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogits_processor\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mLogitsProcessor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/vllm/model_executor/layers/activation.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunctional\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mvllm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_C\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mops\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mvllm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel_executor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mquantization\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mQuantizationConfig\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m from vllm.model_executor.parallel_utils.parallel_state import (\n",
            "\u001b[0;31mImportError\u001b[0m: libcuda.so.1: cannot open shared object file: No such file or directory",
            "",
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"
          ],
          "errorDetails": {
            "actions": [
              {
                "action": "open_url",
                "actionText": "Open Examples",
                "url": "/notebooks/snippets/importing_libraries.ipynb"
              }
            ]
          }
        }
      ]
    }
  ]
}