forked from meta-llama/llama-recipes
-
Notifications
You must be signed in to change notification settings - Fork 0
/
test_finetuning.py
169 lines (122 loc) · 5.62 KB
/
test_finetuning.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
# Copyright (c) Meta Platforms, Inc. and affiliates.
# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement.
import pytest
from pytest import approx
from unittest.mock import patch
import torch
from torch.optim import AdamW
from torch.utils.data.dataloader import DataLoader
from torch.utils.data.sampler import BatchSampler
from llama_recipes.finetuning import main
from llama_recipes.data.sampler import LengthBasedBatchSampler
def get_fake_dataset():
return [{
"input_ids":[1],
"attention_mask":[1],
"labels":[1],
}]
@patch('llama_recipes.finetuning.train')
@patch('llama_recipes.finetuning.LlamaForCausalLM.from_pretrained')
@patch('llama_recipes.finetuning.LlamaTokenizer.from_pretrained')
@patch('llama_recipes.finetuning.get_preprocessed_dataset')
@patch('llama_recipes.finetuning.optim.AdamW')
@patch('llama_recipes.finetuning.StepLR')
def test_finetuning_no_validation(step_lr, optimizer, get_dataset, tokenizer, get_model, train):
kwargs = {"run_validation": False}
get_dataset.return_value = get_fake_dataset()
main(**kwargs)
assert train.call_count == 1
args, kwargs = train.call_args
train_dataloader = args[1]
eval_dataloader = args[2]
assert isinstance(train_dataloader, DataLoader)
assert eval_dataloader is None
if torch.cuda.is_available():
assert get_model.return_value.to.call_count == 1
assert get_model.return_value.to.call_args.args[0] == "cuda"
else:
assert get_model.return_value.to.call_count == 0
@patch('llama_recipes.finetuning.train')
@patch('llama_recipes.finetuning.LlamaForCausalLM.from_pretrained')
@patch('llama_recipes.finetuning.LlamaTokenizer.from_pretrained')
@patch('llama_recipes.finetuning.get_preprocessed_dataset')
@patch('llama_recipes.finetuning.optim.AdamW')
@patch('llama_recipes.finetuning.StepLR')
def test_finetuning_with_validation(step_lr, optimizer, get_dataset, tokenizer, get_model, train):
kwargs = {"run_validation": True}
get_dataset.return_value = get_fake_dataset()
main(**kwargs)
assert train.call_count == 1
args, kwargs = train.call_args
train_dataloader = args[1]
eval_dataloader = args[2]
assert isinstance(train_dataloader, DataLoader)
assert isinstance(eval_dataloader, DataLoader)
if torch.cuda.is_available():
assert get_model.return_value.to.call_count == 1
assert get_model.return_value.to.call_args.args[0] == "cuda"
else:
assert get_model.return_value.to.call_count == 0
@patch('llama_recipes.finetuning.train')
@patch('llama_recipes.finetuning.LlamaForCausalLM.from_pretrained')
@patch('llama_recipes.finetuning.LlamaTokenizer.from_pretrained')
@patch('llama_recipes.finetuning.get_preprocessed_dataset')
@patch('llama_recipes.finetuning.generate_peft_config')
@patch('llama_recipes.finetuning.get_peft_model')
@patch('llama_recipes.finetuning.optim.AdamW')
@patch('llama_recipes.finetuning.StepLR')
def test_finetuning_peft(step_lr, optimizer, get_peft_model, gen_peft_config, get_dataset, tokenizer, get_model, train):
kwargs = {"use_peft": True}
get_dataset.return_value = get_fake_dataset()
main(**kwargs)
if torch.cuda.is_available():
assert get_model.return_value.to.call_count == 1
assert get_model.return_value.to.call_args.args[0] == "cuda"
else:
assert get_model.return_value.to.call_count == 0
assert get_peft_model.return_value.print_trainable_parameters.call_count == 1
@patch('llama_recipes.finetuning.train')
@patch('llama_recipes.finetuning.LlamaForCausalLM.from_pretrained')
@patch('llama_recipes.finetuning.LlamaTokenizer.from_pretrained')
@patch('llama_recipes.finetuning.get_preprocessed_dataset')
@patch('llama_recipes.finetuning.get_peft_model')
@patch('llama_recipes.finetuning.StepLR')
def test_finetuning_weight_decay(step_lr, get_peft_model, get_dataset, tokenizer, get_model, train, mocker):
kwargs = {"weight_decay": 0.01}
get_dataset.return_value = get_fake_dataset()
model = mocker.MagicMock(name="Model")
model.parameters.return_value = [torch.ones(1,1)]
get_model.return_value = model
main(**kwargs)
assert train.call_count == 1
args, kwargs = train.call_args
optimizer = args[4]
print(optimizer.state_dict())
assert isinstance(optimizer, AdamW)
assert optimizer.state_dict()["param_groups"][0]["weight_decay"] == approx(0.01)
@patch('llama_recipes.finetuning.train')
@patch('llama_recipes.finetuning.LlamaForCausalLM.from_pretrained')
@patch('llama_recipes.finetuning.LlamaTokenizer.from_pretrained')
@patch('llama_recipes.finetuning.get_preprocessed_dataset')
@patch('llama_recipes.finetuning.optim.AdamW')
@patch('llama_recipes.finetuning.StepLR')
def test_batching_strategy(step_lr, optimizer, get_dataset, tokenizer, get_model, train):
kwargs = {"batching_strategy": "packing"}
get_dataset.return_value = get_fake_dataset()
main(**kwargs)
assert train.call_count == 1
args, kwargs = train.call_args
train_dataloader, eval_dataloader = args[1:3]
assert isinstance(train_dataloader.batch_sampler, BatchSampler)
assert isinstance(eval_dataloader.batch_sampler, BatchSampler)
kwargs["batching_strategy"] = "padding"
train.reset_mock()
main(**kwargs)
assert train.call_count == 1
args, kwargs = train.call_args
train_dataloader, eval_dataloader = args[1:3]
assert isinstance(train_dataloader.batch_sampler, LengthBasedBatchSampler)
assert isinstance(eval_dataloader.batch_sampler, LengthBasedBatchSampler)
kwargs["batching_strategy"] = "none"
with pytest.raises(ValueError):
main(**kwargs)