use axolotl contribs for fix_untrained_tokens

requirements.txt

@@ -60,3 +60,5 @@ antlr4-python3-runtime==4.13.2
 
 torchao==0.7.0
 schedulefree==1.3.0
+
+axolotl-contribs-lgpl==0.0.1b2

src/axolotl/__init__.py

@@ -1,3 +1,7 @@
 """Axolotl - Train and fine-tune large language models"""
 
+import pkgutil
+
+__path__ = pkgutil.extend_path(__path__, __name__)  # Make this a namespace package
+
 __version__ = "0.6.0"

src/axolotl/train.py

@@ -19,7 +19,9 @@
 from transformers.integrations.deepspeed import is_deepspeed_zero3_enabled
 
 from axolotl.common.cli import TrainerCliArgs
-from axolotl.core.tokenizer_utils import fix_untrained_tokens
+from axolotl.contribs.lgpl.unsloth import (  # pylint: disable = no-name-in-module
+    fix_untrained_tokens,
+)
 from axolotl.logging_config import configure_logging
 from axolotl.utils.dict import DictDefault
 from axolotl.utils.freeze import freeze_layers_except

tests/e2e/test_llama.py

@@ -4,7 +4,6 @@
 
 import logging
 import os
-import unittest
 from pathlib import Path
 
 from axolotl.cli import load_datasets
@@ -13,18 +12,15 @@
 from axolotl.utils.config import normalize_config
 from axolotl.utils.dict import DictDefault
 
-from .utils import with_temp_dir
-
 LOG = logging.getLogger("axolotl.tests.e2e")
 os.environ["WANDB_DISABLED"] = "true"
 
 
-class TestLlama(unittest.TestCase):
+class TestLlama:
     """
     Test case for Llama models
     """
 
-    @with_temp_dir
     def test_fft_trust_remote_code(self, temp_dir):
         # pylint: disable=duplicate-code
         cfg = DictDefault(
@@ -46,7 +42,8 @@ def test_fft_trust_remote_code(self, temp_dir):
                     },
                 ],
                 "num_epochs": 1,
-                "micro_batch_size": 8,
+                "max_steps": 5,
+                "micro_batch_size": 2,
                 "gradient_accumulation_steps": 1,
                 "output_dir": temp_dir,
                 "learning_rate": 0.00001,
@@ -64,3 +61,46 @@ def test_fft_trust_remote_code(self, temp_dir):
 
         train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
         assert (Path(temp_dir) / "model.safetensors").exists()
+
+    def test_fix_untrained_tokens(self, temp_dir):
+        # pylint: disable=duplicate-code
+        cfg = DictDefault(
+            {
+                "base_model": "HuggingFaceTB/SmolLM2-135M",
+                "fix_untrained_tokens": True,
+                "sequence_len": 512,
+                "val_set_size": 0.0,
+                "special_tokens": {
+                    "pad_token": "<|endoftext|>",
+                },
+                "chat_template": "chatml",
+                "datasets": [
+                    {
+                        "path": "mlabonne/FineTome-100k",
+                        "type": "chat_template",
+                        "split": "train[:10%]",
+                        "field_messages": "conversations",
+                        "message_field_role": "from",
+                        "message_field_content": "value",
+                    },
+                ],
+                "num_epochs": 1,
+                "max_steps": 5,
+                "micro_batch_size": 1,
+                "gradient_accumulation_steps": 1,
+                "output_dir": temp_dir,
+                "learning_rate": 0.00001,
+                "optimizer": "adamw_8bit",
+                "lr_scheduler": "cosine",
+                "flash_attention": True,
+                "sample_packing": True,
+                "bf16": True,
+                "save_safetensors": True,
+            }
+        )
+        normalize_config(cfg)
+        cli_args = TrainerCliArgs()
+        dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
+
+        train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
+        assert (Path(temp_dir) / "model.safetensors").exists()