initial hydra recipe

recipes/__init__.py

@@ -5,7 +5,7 @@
 # LICENSE file in the root directory of this source tree.
 
 
-_RECIPE_LIST = ["full_finetune", "lora_finetune", "alpaca_generate"]
+_RECIPE_LIST = ["full_finetune", "lora_finetune", "alpaca_generate", "full_finetune_hydra"]
 _CONFIG_LISTS = {
     "full_finetune": ["alpaca_llama2_full_finetune"],
     "lora_finetune": ["alpaca_llama2_lora_finetune"],

recipes/configs/alpaca_llama2_full_finetune_hydra.yaml

@@ -0,0 +1,46 @@
+# Runs the full_finetune.py recipe using FullFinetuneParams
+#
+# To launch, run the following command from root:
+#    tune --nnodes 1 --nproc_per_node 1 --config alpaca_llama2_full_finetune --override model_checkpoint=<your_checkpoint_dir> ...
+
+# Dataset and Dataloader
+dataset:
+  _target_: torchtune.datasets.AlpacaDataset
+  train_on_input: True
+
+seed: null
+shuffle: True
+
+# Model Arguments
+model:
+  _target_: torchtune.models.llama2_7b
+
+model_checkpoint: /tmp/llama2-7b
+tokenizer:
+  _target_: torchtune.models.llama2_tokenizer
+  path: /tmp/tokenizer.model
+
+# Fine-tuning arguments
+batch_size: 2
+epochs: 3
+optimizer:
+  _target_: torch.optim.SGD
+  lr: 2e-5
+max_steps_per_epoch: null
+gradient_accumulation_steps: 1
+log_every_n_steps: null
+run_generation: null
+
+loss:
+  _target_: torch.nn.CrossEntropyLoss
+
+output_dir: /tmp/alpaca-llama2-finetune
+device: cuda
+dtype: fp32
+enable_fsdp: True
+enable_activation_checkpointing: True
+cpu_offload: False
+resume_from_checkpoint: False
+metric_logger:
+  _target_: torchtune.utils.metric_logging.DiskLogger
+  log_dir: ${output_dir}