From d3081675557634c38294e602b24f73d91334947d Mon Sep 17 00:00:00 2001 From: Erland366 Date: Tue, 19 Nov 2024 01:50:45 +0400 Subject: [PATCH 1/8] HfFileSystem bug where it must be forward slash --- unsloth/models/loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 7a6322d24..725875fa9 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -228,7 +228,7 @@ def from_pretrained( exist_config = os.path.exists(os.path.join(model_name, "config.json")) both_exist = exist_adapter_config and exist_config else: - files = HfFileSystem(token = token).glob(os.path.join(model_name, "*.json")) + files = HfFileSystem(token = token).glob(f"{model_name}/*.json") files = (os.path.split(x)[-1] for x in files) if sum(x == "adapter_config.json" or x == "config.json" for x in files) >= 2: both_exist = True From 2d76d9ae94b219e0f3c1603141e0068e70026376 Mon Sep 17 00:00:00 2001 From: Erland366 Date: Wed, 20 Nov 2024 04:16:10 +0400 Subject: [PATCH 2/8] Enhance Kaggle environment support by updating temporary directory handling and adding zip functionality for saved models --- unsloth/save.py | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/unsloth/save.py b/unsloth/save.py index b4c6b499c..e82fad1c9 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -439,7 +439,7 @@ def unsloth_save_model( print("Saved to https://huggingface.co/" + save_pretrained_settings["save_directory"]) pass - print(" Done.") + print(" Done. Model were saved at " + save_pretrained_settings["save_directory"]) return save_directory, None pass @@ -1120,7 +1120,7 @@ def save_to_gguf( # Check if quantization succeeded! if not os.path.isfile(final_location): if IS_KAGGLE_ENVIRONMENT: - if not Path(final_location).resolve().is_relative_to(Path('/tmp').resolve()): + if not Path(final_location).resolve().is_relative_to(Path(KAGGLE_TMP).resolve()): raise RuntimeError( f"Unsloth: Quantization failed for {final_location}\n"\ "You are in a Kaggle environment, which might be the reason this is failing.\n"\ @@ -1171,7 +1171,7 @@ def save_to_gguf( # Check if quantization succeeded! if not os.path.isfile(final_location): if IS_KAGGLE_ENVIRONMENT: - if not Path(final_location).resolve().is_relative_to(Path('/tmp').resolve()): + if not Path(final_location).resolve().is_relative_to(Path(KAGGLE_TMP).resolve()): raise RuntimeError( f"Unsloth: Quantization failed for {final_location}\n"\ "You are in a Kaggle environment, which might be the reason this is failing.\n"\ @@ -1612,6 +1612,9 @@ def unsloth_save_pretrained_gguf( del arguments["quantization_method"] del arguments["first_conversion"] + if IS_KAGGLE_ENVIRONMENT: + arguments["save_directory"] = os.path.join(KAGGLE_TMP, save_directory) + # Fix tokenizer adding an extra BOS token at the front fix_bos_token, old_chat_template = fix_tokenizer_bos_token(tokenizer) @@ -1703,6 +1706,29 @@ def unsloth_save_pretrained_gguf( ) pass + if IS_KAGGLE_ENVIRONMENT: + print( + "Unsloth: We zipped the file for you to download since we store it in the Kaggle", + f"{KAGGLE_TMP} directory.\n"\ + ) + from zipfile import ZipFile, ZIP_DEFLATED + + list_of_files = list(all_file_locations) + if modelfile_location is not None: + list_of_files.append(modelfile_location) + print(f"Unsloth: Zipping {list_of_files}...") + with ZipFile(os.path.join(KAGGLE_TMP, "unsloth_gguf.zip"), "w", compression=ZIP_DEFLATED) as zip_file: + for file in list_of_files: + try: + zip_file.write(file) + except FileNotFoundError as _: + logger.warning(f"Unsloth: file {file} not found. Skipping...") + pass + + # Now expose the zipfile to user + from IPython.display import FileLink + FileLink(os.path.join(KAGGLE_TMP, "unsloth_gguf.zip")) + if push_to_hub: print("Unsloth: Uploading GGUF to Huggingface Hub...") From 7e63aca2177ae9eff9d94c9efabd08d8707989c7 Mon Sep 17 00:00:00 2001 From: Erland366 Date: Wed, 20 Nov 2024 05:09:34 +0400 Subject: [PATCH 3/8] Refactor Kaggle environment file handling to display individual file links instead of zipping --- unsloth/save.py | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/unsloth/save.py b/unsloth/save.py index e82fad1c9..c86dbf35b 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -1707,27 +1707,17 @@ def unsloth_save_pretrained_gguf( pass if IS_KAGGLE_ENVIRONMENT: - print( - "Unsloth: We zipped the file for you to download since we store it in the Kaggle", - f"{KAGGLE_TMP} directory.\n"\ - ) - from zipfile import ZipFile, ZIP_DEFLATED - list_of_files = list(all_file_locations) if modelfile_location is not None: list_of_files.append(modelfile_location) - print(f"Unsloth: Zipping {list_of_files}...") - with ZipFile(os.path.join(KAGGLE_TMP, "unsloth_gguf.zip"), "w", compression=ZIP_DEFLATED) as zip_file: - for file in list_of_files: - try: - zip_file.write(file) - except FileNotFoundError as _: - logger.warning(f"Unsloth: file {file} not found. Skipping...") - pass - # Now expose the zipfile to user - from IPython.display import FileLink - FileLink(os.path.join(KAGGLE_TMP, "unsloth_gguf.zip")) + from IPython.display import FileLink, display + + for file_location in list_of_files: + if file_location is not None: + display(FileLink(file_location)) + + logger.info("Unsloth: Click the above links to download the files.") if push_to_hub: print("Unsloth: Uploading GGUF to Huggingface Hub...") From 4d25e48f2ab5f80bad51beabf861a82f8a174bee Mon Sep 17 00:00:00 2001 From: Erland366 Date: Wed, 20 Nov 2024 05:12:49 +0400 Subject: [PATCH 4/8] Update save directory handling for Kaggle environment --- unsloth/save.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/unsloth/save.py b/unsloth/save.py index c86dbf35b..aaf585b12 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -1806,6 +1806,9 @@ def unsloth_push_to_hub_gguf( del arguments["quantization_method"] del arguments["first_conversion"] + if IS_KAGGLE_ENVIRONMENT: + arguments["save_directory"] = os.path.join(KAGGLE_TMP, arguments["save_directory"]) + # Fix tokenizer adding an extra BOS token at the front fix_bos_token, old_chat_template = fix_tokenizer_bos_token(tokenizer) From 563417603551e1d38c0c66f1d2b54afc9bdb522c Mon Sep 17 00:00:00 2001 From: Erland366 Date: Wed, 20 Nov 2024 05:41:53 +0400 Subject: [PATCH 5/8] Add link formatting for Kaggle environment in unsloth_push_to_hub_gguf --- unsloth/save.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/unsloth/save.py b/unsloth/save.py index aaf585b12..6a4d22b53 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -1906,6 +1906,10 @@ def unsloth_push_to_hub_gguf( if username not in new_save_directory else \ new_save_directory.lstrip('/.') + if IS_KAGGLE_ENVIRONMENT: + # Take last 2 parts of the link + link = "/".join(link.split("/")[-2:]) + print(f"Saved GGUF to https://huggingface.co/{link}") pass @@ -1915,6 +1919,10 @@ def unsloth_push_to_hub_gguf( self, repo_id, token, "GGUF converted", "gguf", modelfile_location, old_username, private, ) + if IS_KAGGLE_ENVIRONMENT: + # Take last 2 parts of the link + link = "/".join(link.split("/")[-2:]) + print(f"Saved Ollama Modelfile to https://huggingface.co/{link}") pass From 58c0333fb902c43a3215a9739261d88aabf6a640 Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Sat, 7 Dec 2024 14:24:48 -0800 Subject: [PATCH 6/8] Update README.md Llama 3.3 + Reddit --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b47741986..0b3b50583 100644 --- a/README.md +++ b/README.md @@ -41,15 +41,16 @@ All notebooks are **beginner friendly**! Add your dataset, click "Run All", and - Click [here](https://docs.unsloth.ai/) for detailed documentation for Unsloth. ## 🦥 Unsloth.ai News +- 📣 NEW! [Llama 3.3 (70B)](https://huggingface.co/collections/unsloth/llama-33-all-versions-67535d7d994794b9d7cf5e9f), Meta's latest model is now supported. - 📣 NEW! Introducing Unsloth [Dynamic 4-bit Quantization](https://unsloth.ai/blog/dynamic-4bit)! We dynamically opt not to quantize certain parameters and this greatly increases accuracy while only using <10% more VRAM than BnB 4-bit. See our collection on [Hugging Face here.](https://huggingface.co/collections/unsloth/unsloth-4-bit-dynamic-quants-67503bb873f89e15276c44e7) - 📣 NEW! [Vision models](https://unsloth.ai/blog/vision) now supported! [Llama 3.2 Vision (11B)](https://colab.research.google.com/drive/1j0N4XTY1zXXy7mPAhOC1_gMYZ2F2EBlk?usp=sharing), [Qwen 2.5 VL (7B)](https://colab.research.google.com/drive/1whHb54GNZMrNxIsi2wm2EY_-Pvo2QyKh?usp=sharing) and [Pixtral (12B) 2409](https://colab.research.google.com/drive/1K9ZrdwvZRE96qGkCq_e88FgV3MLnymQq?usp=sharing) - 📣 NEW! Qwen-2.5 including [Coder](https://colab.research.google.com/drive/18sN803sU23XuJV9Q8On2xgqHSer6-UZF?usp=sharing) models are now supported with bugfixes. 14b fits in a Colab GPU! [Qwen 2.5 conversational notebook](https://colab.research.google.com/drive/1qN1CEalC70EO1wGKhNxs1go1W9So61R5?usp=sharing) - 📣 NEW! We found and helped fix a [gradient accumulation bug](https://unsloth.ai/blog/gradient)! Please update Unsloth and transformers. -- 📣 NEW! [Mistral Small 22b notebook](https://colab.research.google.com/drive/1oCEHcED15DzL8xXGU1VTx5ZfOJM8WY01?usp=sharing) finetuning fits in under 16GB of VRAM!
Click for more news - + - 📣 Try out [Chat interface](https://colab.research.google.com/drive/1i-8ESvtLRGNkkUQQr_-z_rcSAIo9c3lM?usp=sharing)! +- 📣 NEW! [Mistral Small 22b notebook](https://colab.research.google.com/drive/1oCEHcED15DzL8xXGU1VTx5ZfOJM8WY01?usp=sharing) finetuning fits in under 16GB of VRAM! - 📣 NEW! [Llama 3.1 8b, 70b](https://colab.research.google.com/drive/1Ys44kVvmeZtnICzWz0xgpRnrIOjZAuxp?usp=sharing) & [Mistral Nemo-12b](https://colab.research.google.com/drive/17d3U-CAIwzmbDRqbZ9NnpHxCkmXB6LZ0?usp=sharing) both Base and Instruct are now supported - 📣 NEW! `pip install unsloth` now works! Head over to [pypi](https://pypi.org/project/unsloth/) to check it out! This allows non git pull installs. Use `pip install unsloth[colab-new]` for non dependency installs. - 📣 NEW! Continued Pretraining [notebook](https://colab.research.google.com/drive/1tEd1FrOXWMnCU9UIvdYhs61tkxdMuKZu?usp=sharing) for other languages like Korean! @@ -66,6 +67,7 @@ All notebooks are **beginner friendly**! Add your dataset, click "Run All", and | 🥇 **Benchmarking** | [Performance Tables](https://github.com/unslothai/unsloth/tree/main#-performance-benchmarking) | 🌐 **Released Models** | [Unsloth Releases](https://docs.unsloth.ai/get-started/all-our-models)| | ✍️ **Blog** | [Read our Blogs](https://unsloth.ai/blog)| +|   **Reddit** | [Join our Reddit page](https://reddit.com/r/unsloth)| ## ⭐ Key Features - All kernels written in [OpenAI's Triton](https://openai.com/research/triton) language. **Manual backprop engine**. From 9c9547d4c546a6ab61d74d86270a8a72834a5ed9 Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Tue, 10 Dec 2024 12:15:03 -0800 Subject: [PATCH 7/8] Update README.md Apple ML Cross Entropy --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 0b3b50583..fcd7d73d1 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ All notebooks are **beginner friendly**! Add your dataset, click "Run All", and ## 🦥 Unsloth.ai News - 📣 NEW! [Llama 3.3 (70B)](https://huggingface.co/collections/unsloth/llama-33-all-versions-67535d7d994794b9d7cf5e9f), Meta's latest model is now supported. +- 📣 NEW! We worked with Apple to add [Cut Cross Entropy](https://arxiv.org/abs/2411.09009). Unsloth now supports 89K context for Meta's Llama 3.3 (70B) on a 80GB GPU - 13x longer than HF+FA2. For Llama 3.1 (8B), Unsloth enables 342K context, surpassing its native 128K support. - 📣 NEW! Introducing Unsloth [Dynamic 4-bit Quantization](https://unsloth.ai/blog/dynamic-4bit)! We dynamically opt not to quantize certain parameters and this greatly increases accuracy while only using <10% more VRAM than BnB 4-bit. See our collection on [Hugging Face here.](https://huggingface.co/collections/unsloth/unsloth-4-bit-dynamic-quants-67503bb873f89e15276c44e7) - 📣 NEW! [Vision models](https://unsloth.ai/blog/vision) now supported! [Llama 3.2 Vision (11B)](https://colab.research.google.com/drive/1j0N4XTY1zXXy7mPAhOC1_gMYZ2F2EBlk?usp=sharing), [Qwen 2.5 VL (7B)](https://colab.research.google.com/drive/1whHb54GNZMrNxIsi2wm2EY_-Pvo2QyKh?usp=sharing) and [Pixtral (12B) 2409](https://colab.research.google.com/drive/1K9ZrdwvZRE96qGkCq_e88FgV3MLnymQq?usp=sharing) - 📣 NEW! Qwen-2.5 including [Coder](https://colab.research.google.com/drive/18sN803sU23XuJV9Q8On2xgqHSer6-UZF?usp=sharing) models are now supported with bugfixes. 14b fits in a Colab GPU! [Qwen 2.5 conversational notebook](https://colab.research.google.com/drive/1qN1CEalC70EO1wGKhNxs1go1W9So61R5?usp=sharing) @@ -483,7 +484,20 @@ You can cite the Unsloth repo as follows: } ``` +### Citation +If you would like to cite Unsloth, you can use the following format: +``` +@misc{han2023unsloth, + author = {Daniel Han and Michael Han}, + title = {Unsloth AI: Finetune LLMs 2x faster 2-5x faster with 80\% less memory}, + year = {2023}, + url = {https://github.com/unslothai/unsloth}, + note = {GitHub repository} +} +``` + ### Thank You to +- [Erik](https://github.com/erikwijmans) for his help adding [Apple's ML Cross Entropy](https://github.com/apple/ml-cross-entropy) in Unsloth - [HuyNguyen-hust](https://github.com/HuyNguyen-hust) for making [RoPE Embeddings 28% faster](https://github.com/unslothai/unsloth/pull/238) - [RandomInternetPreson](https://github.com/RandomInternetPreson) for confirming WSL support - [152334H](https://github.com/152334H) for experimental DPO support From 138e9b9a8470974dcf7468fba6145e11ed5b1552 Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Tue, 10 Dec 2024 14:43:27 -0800 Subject: [PATCH 8/8] Update README.md Removing double citation --- README.md | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/README.md b/README.md index fcd7d73d1..6bff98cbd 100644 --- a/README.md +++ b/README.md @@ -472,7 +472,7 @@ Two Tesla T4s on Kaggle ![](https://i.ibb.co/sJ7RhGG/image-41.png)
-### Citing +### Citation You can cite the Unsloth repo as follows: ```bibtex @@ -484,18 +484,6 @@ You can cite the Unsloth repo as follows: } ``` -### Citation -If you would like to cite Unsloth, you can use the following format: -``` -@misc{han2023unsloth, - author = {Daniel Han and Michael Han}, - title = {Unsloth AI: Finetune LLMs 2x faster 2-5x faster with 80\% less memory}, - year = {2023}, - url = {https://github.com/unslothai/unsloth}, - note = {GitHub repository} -} -``` - ### Thank You to - [Erik](https://github.com/erikwijmans) for his help adding [Apple's ML Cross Entropy](https://github.com/apple/ml-cross-entropy) in Unsloth - [HuyNguyen-hust](https://github.com/HuyNguyen-hust) for making [RoPE Embeddings 28% faster](https://github.com/unslothai/unsloth/pull/238)