Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/main' into dev
Browse files Browse the repository at this point in the history
  • Loading branch information
hbaghramyan committed Jul 11, 2024
2 parents 075d71c + 4f0a107 commit 891eb7e
Show file tree
Hide file tree
Showing 6 changed files with 262 additions and 21 deletions.
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ Alternatively, you can view this and other files on GitHub at [https://github.co
<br>
&nbsp

Shown below is a mental model summarizing the contents covered in this book.
The mental model below summarizes the contents covered in this book.

<img src="https://sebastianraschka.com/images/LLMs-from-scratch-images/mental-model.jpg" width="650px">

Expand Down
21 changes: 5 additions & 16 deletions ch02/01_main-chapter-code/ch02.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,6 @@
"source": [
"from importlib.metadata import version\n",
"\n",
"import tiktoken\n",
"import torch\n",
"\n",
"print(\"torch version:\", version(\"torch\"))\n",
"print(\"tiktoken version:\", version(\"tiktoken\"))"
]
Expand Down Expand Up @@ -870,7 +867,7 @@
" def decode(self, ids):\n",
" text = \" \".join([self.int_to_str[i] for i in ids])\n",
" # Replace spaces before the specified punctuations\n",
" text = re.sub(r'\\s+([,.?!\"()\\'])', r'\\1', text)\n",
" text = re.sub(r'\\s+([,.:;?!\"()\\'])', r'\\1', text)\n",
" return text"
]
},
Expand All @@ -885,7 +882,7 @@
{
"cell_type": "code",
"execution_count": 22,
"id": "daaf8cf8-d1a3-4640-a7c8-76fc398e7e0c",
"id": "4133c502-18ac-4412-9f43-01caf4efa3dc",
"metadata": {},
"outputs": [
{
Expand All @@ -910,7 +907,7 @@
{
"cell_type": "code",
"execution_count": 23,
"id": "ddfe7346-398d-4bf8-99f1-5b071244ce95",
"id": "7ed395fe-dc1b-4ed2-b85b-457cc35aab60",
"metadata": {},
"outputs": [
{
Expand All @@ -931,7 +928,7 @@
{
"cell_type": "code",
"execution_count": 24,
"id": "0c350ff6-2734-4e84-9ec7-d578baa4ae1b",
"id": "059367f9-7a60-4c0d-8a00-7c4c766d0ebc",
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -1885,14 +1882,6 @@
"\n",
"See [./exercise-solutions.ipynb](./exercise-solutions.ipynb) for the exercise solutions."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c6aa436b-c9ff-4f01-9a58-5c377b3700a5",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
Expand All @@ -1911,7 +1900,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.4"
"version": "3.10.11"
}
},
"nbformat": 4,
Expand Down
3 changes: 3 additions & 0 deletions ch06/01_main-chapter-code/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,8 @@

### Optional Code

[load-finetuned-model.ipynb](load-finetuned-model.ipynb) is a standalone Jupyter notebook to load the finetuned model we created in this chapter


- [gpt_class_finetune.py](gpt_class_finetune.py) is a standalone Python script file with the code that we implemented in [ch06.ipynb](ch06.ipynb) to finetune the GPT model (you can think of it as a chapter summary)

4 changes: 2 additions & 2 deletions ch06/01_main-chapter-code/ch06.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -2326,7 +2326,7 @@
}
],
"source": [
"model_state_dict = torch.load(\"review_classifier.pth\")\n",
"model_state_dict = torch.load(\"review_classifier.pth\", map_location=device)\n",
"model.load_state_dict(model_state_dict)"
]
},
Expand Down Expand Up @@ -2372,7 +2372,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.4"
"version": "3.12.2"
}
},
"nbformat": 4,
Expand Down
249 changes: 249 additions & 0 deletions ch06/01_main-chapter-code/load-finetuned-model.ipynb
Original file line number Diff line number Diff line change
@@ -0,0 +1,249 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "1545a16b-bc8d-4e49-b9a6-db6631e7483d",
"metadata": {},
"source": [
"<table style=\"width:100%\">\n",
"<tr>\n",
"<td style=\"vertical-align:middle; text-align:left;\">\n",
"<font size=\"2\">\n",
"Supplementary code for the <a href=\"http://mng.bz/orYv\">Build a Large Language Model From Scratch</a> book by <a href=\"https://sebastianraschka.com\">Sebastian Raschka</a><br>\n",
"<br>Code repository: <a href=\"https://github.com/rasbt/LLMs-from-scratch\">https://github.com/rasbt/LLMs-from-scratch</a>\n",
"</font>\n",
"</td>\n",
"<td style=\"vertical-align:middle; text-align:left;\">\n",
"<a href=\"http://mng.bz/orYv\"><img src=\"https://sebastianraschka.com/images/LLMs-from-scratch-images/cover-small.webp\" width=\"100px\"></a>\n",
"</td>\n",
"</tr>\n",
"</table>"
]
},
{
"cell_type": "markdown",
"id": "f3f83194-82b9-4478-9550-5ad793467bd0",
"metadata": {},
"source": [
"# Load And Use Finetuned Model"
]
},
{
"cell_type": "markdown",
"id": "466b564e-4fd5-4d76-a3a1-63f9f0993b7e",
"metadata": {},
"source": [
"This notebook contains minimal code to load the finetuned model that was created and saved in chapter 6 via [ch06.ipynb](ch06.ipynb)."
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "fd80e5f5-0f79-4a6c-bf31-2026e7d30e52",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"tiktoken version: 0.6.0\n",
"torch version: 2.2.2\n"
]
}
],
"source": [
"from importlib.metadata import version\n",
"\n",
"pkgs = [\n",
" \"tiktoken\", # Tokenizer\n",
" \"torch\", # Deep learning library\n",
"]\n",
"for p in pkgs:\n",
" print(f\"{p} version: {version(p)}\")"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "ed86d6b7-f32d-4601-b585-a2ea3dbf7201",
"metadata": {},
"outputs": [],
"source": [
"from pathlib import Path\n",
"\n",
"finetuned_model_path = Path(\"review_classifier.pth\")\n",
"if not finetuned_model_path.exists():\n",
" print(\n",
" f\"Could not find '{finetuned_model_path}'.\\n\"\n",
" \"Run the `ch06.ipynb` notebook to finetune and save the finetuned model.\"\n",
" )"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "fb02584a-5e31-45d5-8377-794876907bc6",
"metadata": {},
"outputs": [],
"source": [
"from previous_chapters import GPTModel\n",
"\n",
"\n",
"BASE_CONFIG = {\n",
" \"vocab_size\": 50257, # Vocabulary size\n",
" \"context_length\": 1024, # Context length\n",
" \"drop_rate\": 0.0, # Dropout rate\n",
" \"qkv_bias\": True # Query-key-value bias\n",
"}\n",
"\n",
"model_configs = {\n",
" \"gpt2-small (124M)\": {\"emb_dim\": 768, \"n_layers\": 12, \"n_heads\": 12},\n",
" \"gpt2-medium (355M)\": {\"emb_dim\": 1024, \"n_layers\": 24, \"n_heads\": 16},\n",
" \"gpt2-large (774M)\": {\"emb_dim\": 1280, \"n_layers\": 36, \"n_heads\": 20},\n",
" \"gpt2-xl (1558M)\": {\"emb_dim\": 1600, \"n_layers\": 48, \"n_heads\": 25},\n",
"}\n",
"\n",
"CHOOSE_MODEL = \"gpt2-small (124M)\"\n",
"\n",
"BASE_CONFIG.update(model_configs[CHOOSE_MODEL])\n",
"\n",
"# Initialize base model\n",
"model_size = CHOOSE_MODEL.split(\" \")[-1].lstrip(\"(\").rstrip(\")\")\n",
"model = GPTModel(BASE_CONFIG)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "f1ccf2b7-176e-4cfd-af7a-53fb76010b94",
"metadata": {},
"outputs": [],
"source": [
"import torch\n",
"\n",
"# Convert model to classifier as in section 6.5 in ch06.ipynb\n",
"num_classes = 2\n",
"model.out_head = torch.nn.Linear(in_features=BASE_CONFIG[\"emb_dim\"], out_features=num_classes)\n",
"\n",
"# Then load pretrained weights\n",
"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
"model.load_state_dict(torch.load(\"review_classifier.pth\", map_location=device))\n",
"model.eval();"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "a1fd174e-9555-46c5-8780-19b0aa4f26e5",
"metadata": {},
"outputs": [],
"source": [
"import tiktoken\n",
"\n",
"tokenizer = tiktoken.get_encoding(\"gpt2\")"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "2a4c0129-efe5-46e9-bb90-ba08d407c1a2",
"metadata": {},
"outputs": [],
"source": [
"# This function was implemented in ch06.ipynb\n",
"def classify_review(text, model, tokenizer, device, max_length=None, pad_token_id=50256):\n",
" model.eval()\n",
"\n",
" # Prepare inputs to the model\n",
" input_ids = tokenizer.encode(text)\n",
" supported_context_length = model.pos_emb.weight.shape[1]\n",
"\n",
" # Truncate sequences if they too long\n",
" input_ids = input_ids[:min(max_length, supported_context_length)]\n",
"\n",
" # Pad sequences to the longest sequence\n",
" input_ids += [pad_token_id] * (max_length - len(input_ids))\n",
" input_tensor = torch.tensor(input_ids, device=device).unsqueeze(0) # add batch dimension\n",
"\n",
" # Model inference\n",
" with torch.no_grad():\n",
" logits = model(input_tensor)[:, -1, :] # Logits of the last output token\n",
" predicted_label = torch.argmax(logits, dim=-1).item()\n",
"\n",
" # Return the classified result\n",
" return \"spam\" if predicted_label == 1 else \"not spam\""
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "1e26862c-10b5-4a0f-9dd6-b6ddbad2fc3f",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"spam\n"
]
}
],
"source": [
"text_1 = (\n",
" \"You are a winner you have been specially\"\n",
" \" selected to receive $1000 cash or a $2000 award.\"\n",
")\n",
"\n",
"print(classify_review(\n",
" text_1, model, tokenizer, device, max_length=120\n",
"))"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "78472e05-cb4e-4ec4-82e8-23777aa90cf8",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"not spam\n"
]
}
],
"source": [
"text_2 = (\n",
" \"Hey, just wanted to check if we're still on\"\n",
" \" for dinner tonight? Let me know!\"\n",
")\n",
"\n",
"print(classify_review(\n",
" text_2, model, tokenizer, device, max_length=120\n",
"))"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.2"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
4 changes: 2 additions & 2 deletions ch07/01_main-chapter-code/load-finetuned-model.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@
"if not finetuned_model_path.exists():\n",
" print(\n",
" f\"Could not find '{finetuned_model_path}'.\\n\"\n",
" \"Run the `ch07.ipynb` notebook to finetune and save finetuned model.\"\n",
" \"Run the `ch07.ipynb` notebook to finetune and save the finetuned model.\"\n",
" )"
]
},
Expand Down Expand Up @@ -207,7 +207,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.4"
"version": "3.12.2"
}
},
"nbformat": 4,
Expand Down

0 comments on commit 891eb7e

Please sign in to comment.