remote_inference_gateway-users.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LOGIN AND FETCH TOKEN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Please go to this URL and login:\n",
      "\n",
      "https://auth.globus.org/v2/oauth2/authorize?client_id=58fdd3bc-e1c3-4ce5-80ea-8d6b87cfb944&redirect_uri=https%3A%2F%2Fauth.globus.org%2Fv2%2Fweb%2Fauth-code&scope=https%3A%2F%2Fauth.globus.org%2Fscopes%2F681c10cc-f684-4540-bcd7-0b4df3bc26ef%2Faction_all&state=_default&response_type=code&code_challenge=ebucvVdS457Amtj69rBsQ24mDj_BOCJ9TxkBCMHn-zs&code_challenge_method=S256&access_type=online\n",
      "\n",
      "Enter the code you just received: ^C\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/adityatanikanti/Codes/inference-endpoints/generate_auth_token.py\", line 29, in <module>\n",
      "    auth_code = input(\"Enter the code you just received: \")\n",
      "                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "KeyboardInterrupt\n"
     ]
    }
   ],
   "source": [
    "!python3 generate_auth_token.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Collect access token to vLLM service\n",
    "auth_code = \"..paste.your.auth.code.here..\"\n",
    "token_response = auth_client.oauth2_exchange_code_for_tokens(auth_code)\n",
    "access_token = token_response.by_resource_server[gateway_client_id][\"access_token\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## INFERENCE USING VLLM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CHAT COMPLETIONS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n",
      "{'server_response': '{\\n    \"id\": \"cmpl-364f2af1491a4a3ebf4d670ac55867e9\",\\n    \"object\": \"chat.completion\",\\n    \"created\": 1721315660,\\n    \"model\": \"meta-llama/Meta-Llama-3-8B-Instruct\",\\n    \"choices\": [\\n        {\\n            \"index\": 0,\\n            \"message\": {\\n                \"role\": \"assistant\",\\n                \"content\": \"RAD51 is a key protein involved in homologous recombination (HR) and DNA repair. It has been shown to interact with numerous proteins to regulate its activity and function. Here is a list of some of the proteins that interact with RAD51:\\\\n\\\\n1. BRCA1: RAD51 interacts with BRCA1, a tumor suppressor protein, to promote HR and DNA repair.\\\\n2. BRCA2: RAD51 also interacts with BRCA2, another tumor suppressor protein, to facilitate HR and DNA repair.\\\\n3. PALB2: PALB2, a protein involved in HR and DNA repair, interacts with RAD51 and BRCA2 to regulate their activity.\\\\n4. FANCD2: FANCD2,\",\\n                \"tool_calls\": []\\n            },\\n            \"logprobs\": null,\\n            \"finish_reason\": \"length\",\\n            \"stop_reason\": null\\n        }\\n    ],\\n    \"usage\": {\\n        \"prompt_tokens\": 18,\\n        \"total_tokens\": 168,\\n        \"completion_tokens\": 150\\n    },\\n    \"response_time\": 4.208064079284668,\\n    \"throughput_tokens_per_second\": 39.923346421226185\\n}'}\n"
     ]
    }
   ],
   "source": [
    "model =  \"meta-llama/Meta-Llama-3-8B-Instruct\"\n",
    "query = \"List all proteins that interact with RAD51\"\n",
    "\n",
    "# URL to the inference gateway\n",
    "url = \"https://data-portal-dev.cels.anl.gov/resource_server/sophia/vllm/v1/chat/completions\"\n",
    "\n",
    "# Add access token to the headers\n",
    "headers = {\n",
    "    \"Authorization\": f\"Bearer {access_token}\",\n",
    "    \"Content-Type\": \"application/json\"\n",
    "}\n",
    "# Prepare the inference request\n",
    "data = {\n",
    "        \"model\": model,\n",
    "        \"messages\":[{\"role\": \"user\", \"content\": query}],\n",
    "}\n",
    "\n",
    "# Convert data into Json\n",
    "data_json = json.dumps(data)\n",
    "\n",
    "# Send the post request to the relay server\n",
    "response = requests.post(url, data=data_json, headers=headers)\n",
    "\n",
    "# Print inference response\n",
    "print(response.status_code)\n",
    "print(response.json())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CHAT COMPLETIONS USING PYTHON OPENAI API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from openai import OpenAI\n",
    " \n",
    "# Set OpenAI's API key and API base to use vLLM's API server.\n",
    "# localurl = \"http://localhost:8000/resource_server/polaris/vllm/v1\"\n",
    "openai_api_key = access_token\n",
    "openai_api_base = \"https://data-portal-dev.cels.anl.gov/resource_server/sophia/vllm/v1\"\n",
    "\n",
    "client = OpenAI(\n",
    "    api_key=openai_api_key,\n",
    "    base_url=openai_api_base,\n",
    ")\n",
    "\n",
    "model=\"meta-llama/Meta-Llama-3-8B-Instruct\"\n",
    "query = \"A detailed description of the biochemical \\\n",
    "            function 5-(hydroxymethyl)furfural/furfural transporter is\"\n",
    "# sampling_params = SamplingParams({\"prompt_logprobs\": 1, \"logprobs\": 1))\n",
    "\n",
    "chat_response = client.chat.completions.create(\n",
    "    model=\"meta-llama/Meta-Llama-3-8B-Instruct\",\n",
    "    logprobs=True,\n",
    "    top_logprobs=1,\n",
    "    messages=[\n",
    "        {\"role\": \"user\", \"content\": query},\n",
    "    ],\n",
    "    max_tokens=2056\n",
    ")\n",
    "print(\"Chat response:\", chat_response)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}