03_inference.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "16bcbd84-d698-4346-aa07-95e13e754e5b",
   "metadata": {},
   "source": [
    "# Running Inference of your FT model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98d8e5cd-cbc7-4dae-bdbb-07b159361a0f",
   "metadata": {
    "tags": []
   },
   "source": [
    "## SageMaker auth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4d21cc52-1b97-4629-9044-4d076e60cb5c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /root/.config/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /root/.config/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /root/.config/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /root/.config/sagemaker/config.yaml\n",
      "sagemaker role arn: arn:aws:iam::372108735839:role/SageMakerExecutionRole\n",
      "sagemaker bucket: sagemaker-us-east-1-372108735839\n",
      "sagemaker session region: us-east-1\n"
     ]
    }
   ],
   "source": [
    "import sagemaker\n",
    "import boto3\n",
    "sess = sagemaker.Session()\n",
    "# sagemaker session bucket -> used for uploading data, models and logs\n",
    "# sagemaker will automatically create this bucket if it not exists\n",
    "sagemaker_session_bucket=None\n",
    "if sagemaker_session_bucket is None and sess is not None:\n",
    "    # set to default bucket if a bucket name is not given\n",
    "    sagemaker_session_bucket = sess.default_bucket()\n",
    "\n",
    "try:\n",
    "    role = sagemaker.get_execution_role()\n",
    "except ValueError:\n",
    "    iam = boto3.client('iam')\n",
    "    role = iam.get_role(RoleName='sagemaker_execution_role')['Role']['Arn']\n",
    "\n",
    "sess = sagemaker.Session(default_bucket=sagemaker_session_bucket)\n",
    "\n",
    "DATASET_S3 = f's3://{sess.default_bucket()}/processed/wandbot/train'\n",
    "\n",
    "print(f\"sagemaker role arn: {role}\")\n",
    "print(f\"sagemaker bucket: {sess.default_bucket()}\")\n",
    "print(f\"sagemaker session region: {sess.boto_region_name}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "726daffc-3498-49e9-9664-a12712004701",
   "metadata": {},
   "source": [
    "## Creating the Endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8ee1bddc-4f68-4055-8824-2b43ce41ed44",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "llm image uri: 763104351884.dkr.ecr.us-east-1.amazonaws.com/huggingface-pytorch-tgi-inference:2.0.1-tgi1.1.0-gpu-py39-cu118-ubuntu20.04\n"
     ]
    }
   ],
   "source": [
    "from sagemaker.huggingface import get_huggingface_llm_image_uri\n",
    "\n",
    "# retrieve the llm image uri\n",
    "llm_image = get_huggingface_llm_image_uri(\n",
    "  \"huggingface\",\n",
    "  version=\"1.1.0\",\n",
    "  session=sess,\n",
    ")\n",
    "\n",
    "# print ecr image uri\n",
    "print(f\"llm image uri: {llm_image}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b47162c-b559-4e19-937c-cece0334331f",
   "metadata": {},
   "source": [
    "Lets create the endpoint with the model stored on S3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "76bd5ae1-5315-4d6d-9563-eef6a6c1eb61",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "model_s3_path = \"s3://sagemaker-us-east-1-372108735839/wandb-qlora-codellama7-2023-10-26-00-01-56-374/output/model/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "80b21176-4820-4c21-887a-bb811981d592",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /root/.config/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /root/.config/sagemaker/config.yaml\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "from sagemaker.huggingface import HuggingFaceModel\n",
    "\n",
    "# s3 path where the model will be uploaded\n",
    "# if you try to deploy the model to a different time add the s3 path here\n",
    "# model_s3_path = huggingface_estimator.model_data[\"S3DataSource\"][\"S3Uri\"]\n",
    "\n",
    "# sagemaker config\n",
    "instance_type = \"ml.g5.2xlarge\"\n",
    "number_of_gpu = 1\n",
    "health_check_timeout = 300\n",
    "\n",
    "# Define Model and Endpoint configuration parameter\n",
    "config = {\n",
    "  'HF_MODEL_ID': \"/opt/ml/model\", # path to where sagemaker stores the model\n",
    "  'SM_NUM_GPUS': json.dumps(number_of_gpu), # Number of GPU used per replica\n",
    "  'MAX_INPUT_LENGTH': json.dumps(2048), # Max length of input text\n",
    "  'MAX_TOTAL_TOKENS': json.dumps(3000), # Max length of the generation (including input text)\n",
    "}\n",
    "\n",
    "# create HuggingFaceModel with the image uri\n",
    "llm_model = HuggingFaceModel(\n",
    "  role=role,\n",
    "  image_uri=llm_image,\n",
    "  model_data={'S3DataSource':{'S3Uri': model_s3_path,'S3DataType': 'S3Prefix','CompressionType': 'None'}},\n",
    "  env=config\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c16e368a-3322-4c23-80ce-b4980113f6ed",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------!"
     ]
    }
   ],
   "source": [
    "# Deploy model to an endpoint\n",
    "# https://sagemaker.readthedocs.io/en/stable/api/inference/model.html#sagemaker.model.Model.deploy\n",
    "llm = llm_model.deploy(\n",
    "  initial_instance_count=1,\n",
    "  instance_type=instance_type,\n",
    "  container_startup_health_check_timeout=health_check_timeout, # 10 minutes to be able to load the model\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "de1dc7c5-33a5-420f-8e62-aa77aaf29c6f",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'huggingface-pytorch-tgi-inference-2023-10-26-11-46-38-157'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "llm.endpoint_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b6876547-1f1e-4bd9-8a90-5eb4c8d7f8f3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "session = sess.boto_session\n",
    "smr = session.client(\"sagemaker-runtime\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "105ef38c-5a9b-4866-9be2-ae105ea8d3a9",
   "metadata": {},
   "source": [
    "## Eval Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "42fc9185-f4b1-4238-9acf-20dfe81b9815",
   "metadata": {},
   "source": [
    "I recommend reading this [excellent blog post](https://www.philschmid.de/sagemaker-falcon-180b) to understand the ins and outs of deploying to SageMaker with Huggingface models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "534bc0df-0fa9-4310-be66-01524e117ff8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install -qqqU wandb datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6ef6f5f9-74d9-4fa3-a1b9-f8977a301d2a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "WANDB_PROJECT = \"aws_llm_workshop\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "3d48aaf2-4fcf-4200-83ea-654b15c156f7",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.15.12"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/root/sagemaker-llama2-wandb/wandb/run-20231026_121855-baqd6znk</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/capecape/aws_llm_workshop/runs/baqd6znk' target=\"_blank\">worthy-forest-23</a></strong> to <a href='https://wandb.ai/capecape/aws_llm_workshop' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/capecape/aws_llm_workshop' target=\"_blank\">https://wandb.ai/capecape/aws_llm_workshop</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/capecape/aws_llm_workshop/runs/baqd6znk' target=\"_blank\">https://wandb.ai/capecape/aws_llm_workshop/runs/baqd6znk</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m:   4 of 4 files downloaded.  \n"
     ]
    }
   ],
   "source": [
    "import wandb\n",
    "import json\n",
    "from datasets import load_from_disk\n",
    "run = wandb.init(project=WANDB_PROJECT, job_type=\"inference\")\n",
    "artifact = run.use_artifact('capecape/aws_llm_workshop/wandbot_eval_dataset:v1', type='dataset')\n",
    "artifact_dir = artifact.download()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "61dfe84d-a3f2-4b5f-b2c0-42a6b9fe7387",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "eval_ds = load_from_disk(artifact_dir)[\"train\"]  # I know, it has a split named train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "5ee07d36-d2b2-4a6b-a8fb-ea55f66c8a5a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "one_sample = eval_ds[0][\"text\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "36907374-9511-4acd-a7d8-516ac0a79435",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# formatted_prompt = format_prompt(prompt, history, system_prompt)\n",
    "import json\n",
    "\n",
    "# hyperparameters for llm\n",
    "parameters = {\n",
    "    \"do_sample\": True,\n",
    "    \"top_p\": 0.9,\n",
    "    \"temperature\": 0.8,\n",
    "    \"max_new_tokens\": 952,\n",
    "    \"repetition_penalty\": 1.03,\n",
    "    \"stop\": [\"[/W&B]\", \"</s>\"],\n",
    "}\n",
    "\n",
    "# hyperparameters for llm\n",
    "payload = {\n",
    "  \"inputs\": one_sample,\n",
    "  \"parameters\": parameters,\n",
    "}\n",
    "\n",
    "# send request to endpoint\n",
    "response = llm.predict(payload)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "72b78364-e6b5-4bbd-940b-9237d430a5fa",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Yes, you can initialize wandb within a specific route function. In your case, `@app.route('/')` would be an ideal place to initialize W&B. Here's a sample code snippet:\\n\\n```python\\nimport wandb\\nfrom fastapi import FastAPI\\n\\napp = FastAPI()\\n\\n@app.get(\"/\")\\ndef read_root():\\n    return {\"Hello\": \"World\"}\\n\\n```\\n\\nIn this code, `app` is a FastAPI instance, `read_root` is a simple route function that returns a dictionary. You can replace `read_root` with your own route function. For instance, you can add `@app.post(\"/\")` to handle HTTP POST requests.\\n\\n\\nIt's important to note that, you should only initialize wandb once for your process. If you initialize it in every route function, it will create multiple runs, which can lead to confusion.\\n[/W&B]\n"
     ]
    }
   ],
   "source": [
    "assistant = response[0][\"generated_text\"]\n",
    "print(assistant)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "5a292cc8-f5d7-4b0a-a278-0fb0002fa64b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from time import perf_counter\n",
    "\n",
    "def call_endpoint(formatted_input, parameters):\n",
    "    \"Call the SM endpoint and parse the output in string format\"\n",
    "    t0 = perf_counter()\n",
    "    payload = {\n",
    "      \"inputs\": formatted_input,\n",
    "      \"parameters\": parameters,\n",
    "    }\n",
    "    response = llm.predict(payload)\n",
    "    total_time = perf_counter() - t0\n",
    "\n",
    "    return response[0][\"generated_text\"], total_time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "5d816664-db91-4659-806f-9977704a8ec3",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('Yes, you can initialize wandb within a specific route function. In fact, that is one of the best practices for initializing wandb. You can do it like this:\\n\\n```python\\nfrom fastapi import FastAPI\\nimport wandb\\n\\napp = FastAPI()\\n\\n@app.on_event(\"startup\")\\nasync def startup():\\n    await wandb.init(...)\\n```\\n\\nHowever, you should initialize wandb beforehand if you want to be able to use `@wandb_log` decorator for logging. Here is an example of how to do that:\\n\\n```python\\nfrom fastapi import FastAPI\\nimport wandb\\n\\napp = FastAPI()\\n\\nwandb.init(...)\\n\\n@app.on_event(\"startup\")\\nasync def startup():\\n    pass\\n```\\n\\nRegarding your issue with long list of empty log items, I would suggest that you make sure you are not logging the same thing multiple times. W&B does not allow logging the same value multiple times within a single run. You can also check the size of your log items by using `wandb.log({\"a\": wandb.Histogram(np.random.random(1000))})`. If the histogram size is large, it might be a sign that you are logging too much data. Also, make sure you are not logging too frequently. W&B has a 10MB limit per `wandb.log` call, so you should try to limit the number of calls to `wandb.log` if you are logging a lot of data. Finally, you can check the size of your run in the UI and remove any unwanted data with `wandb.run.delete_artifact(name)`\\\\n[/W&B]',\n",
       " 12.701498716996866)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "call_endpoint(one_sample, parameters)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf263311-3a0c-4583-9eb7-aae3f5aff7a6",
   "metadata": {},
   "source": [
    "## Saving our eval results to W&B"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "010c7b7a-6c7a-411a-808a-709a7ab618d7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from tqdm.auto import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "b1efab40-2b53-4e89-a578-cbd85144f7ca",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "params_cols = list(parameters.keys())\n",
    "table = wandb.Table(columns=[\"question\", \"original_answer\", \"generated_answer\", \"time(s)\"] + params_cols) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "f8ad2166-e508-49cc-a21c-a71f166124a1",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "307cefa750244ee8a8a48a6e58cd6c46",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/132 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for s in tqdm(eval_ds):\n",
    "    generated_answer, req_time = call_endpoint(s[\"text\"], parameters)\n",
    "    table.add_data(s[\"question\"], s[\"answer\"], generated_answer, req_time, *list(parameters.values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "4cefc701-da4b-46bf-ab7f-cfa4b2ef0bee",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "Waiting for W&B process to finish... <strong style=\"color:green\">(success).</strong>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "wandb: WARNING Source type is set to 'repo' but some required information is missing from the environment. A job will not be created from this run. See https://docs.wandb.ai/guides/launch/create-job\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "026f59aa3e864dc7bde56f938876cc47",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(Label(value='0.350 MB of 0.695 MB uploaded (0.000 MB deduped)\\r'), FloatProgress(value=0.504367…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run <strong style=\"color:#cdcd00\">worthy-forest-23</strong> at: <a href='https://wandb.ai/capecape/aws_llm_workshop/runs/baqd6znk' target=\"_blank\">https://wandb.ai/capecape/aws_llm_workshop/runs/baqd6znk</a><br/>Synced 7 W&B file(s), 1 media file(s), 1 artifact file(s) and 1 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>./wandb/run-20231026_121855-baqd6znk/logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "wandb.log({\"evaluation_answers\": table})\n",
    "wandb.finish()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb0cd080-dc4c-454e-8b19-8ec9423a3018",
   "metadata": {},
   "source": [
    "don't forget to kill the endopoint! "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "983a1771-0a99-4c76-ad09-ccef990bc638",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "llm.delete_model()\n",
    "llm.delete_endpoint()"
   ]
  }
 ],
 "metadata": {
  "availableInstances": [
   {
    "_defaultOrder": 0,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.t3.medium",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 1,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.t3.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 2,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.t3.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 3,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.t3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 4,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 5,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 6,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 7,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 8,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 9,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 10,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 11,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 12,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5d.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 13,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5d.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 14,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5d.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 15,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5d.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 16,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5d.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 17,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5d.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 18,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5d.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 19,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 20,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": true,
    "memoryGiB": 0,
    "name": "ml.geospatial.interactive",
    "supportedImageNames": [
     "sagemaker-geospatial-v1-0"
    ],
    "vcpuNum": 0
   },
   {
    "_defaultOrder": 21,
    "_isFastLaunch": true,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.c5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 22,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.c5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 23,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.c5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 24,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.c5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 25,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 72,
    "name": "ml.c5.9xlarge",
    "vcpuNum": 36
   },
   {
    "_defaultOrder": 26,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 96,
    "name": "ml.c5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 27,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 144,
    "name": "ml.c5.18xlarge",
    "vcpuNum": 72
   },
   {
    "_defaultOrder": 28,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.c5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 29,
    "_isFastLaunch": true,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g4dn.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 30,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g4dn.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 31,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g4dn.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 32,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g4dn.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 33,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g4dn.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 34,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g4dn.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 35,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 61,
    "name": "ml.p3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 36,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 244,
    "name": "ml.p3.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 37,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 488,
    "name": "ml.p3.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 38,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.p3dn.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 39,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.r5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 40,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.r5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 41,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.r5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 42,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.r5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 43,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.r5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 44,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.r5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 45,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 512,
    "name": "ml.r5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 46,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.r5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 47,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 48,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 49,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 50,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 51,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 52,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 53,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.g5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 54,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.g5.48xlarge",
    "vcpuNum": 192
   },
   {
    "_defaultOrder": 55,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 56,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4de.24xlarge",
    "vcpuNum": 96
   }
  ],
  "instance_type": "ml.t3.medium",
  "kernelspec": {
   "display_name": "Python 3 (Data Science 3.0)",
   "language": "python",
   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-east-1:081325390199:image/sagemaker-data-science-310-v1"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}