Skip to content

Commit e9df12c

Browse files
artek0chumakartek0chumak
authored
Add tests for creating finetuning requests (#284)
Add tests for creating finetuning requests
1 parent 10ad24d commit e9df12c

File tree

4 files changed

+287
-17
lines changed

4 files changed

+287
-17
lines changed

‎pyproject.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ build-backend = "poetry.masonry.api"
1212

1313
[tool.poetry]
1414
name = "together"
15-
version = "1.5.2"
15+
version = "1.5.3"
1616
authors = [
1717
"Together AI <[email protected]>"
1818
]

‎src/together/resources/finetune.py

+22-15
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,9 @@ def createFinetuneRequest(
8787
"You must specify either a model or a checkpoint to start a job from, not both"
8888
)
8989

90+
if model is None and from_checkpoint is None:
91+
raise ValueError("You must specify either a model or a checkpoint")
92+
9093
if batch_size == "max":
9194
log_warn_once(
9295
"Starting from together>=1.3.0, "
@@ -96,6 +99,8 @@ def createFinetuneRequest(
9699
warmup_ratio = 0.0
97100

98101
training_type: TrainingType = FullTrainingType()
102+
max_batch_size: int = 0
103+
min_batch_size: int = 0
99104
if lora:
100105
if model_limits.lora_training is None:
101106
raise ValueError("LoRA adapters are not supported for the selected model.")
@@ -108,18 +113,26 @@ def createFinetuneRequest(
108113
lora_trainable_modules=lora_trainable_modules,
109114
)
110115

111-
batch_size = (
112-
batch_size
113-
if batch_size != "max"
114-
else model_limits.lora_training.max_batch_size
115-
)
116+
max_batch_size = model_limits.lora_training.max_batch_size
117+
min_batch_size = model_limits.lora_training.min_batch_size
118+
116119
else:
117120
if model_limits.full_training is None:
118121
raise ValueError("Full training is not supported for the selected model.")
119-
batch_size = (
120-
batch_size
121-
if batch_size != "max"
122-
else model_limits.full_training.max_batch_size
122+
123+
max_batch_size = model_limits.full_training.max_batch_size
124+
min_batch_size = model_limits.full_training.min_batch_size
125+
126+
batch_size = batch_size if batch_size != "max" else max_batch_size
127+
128+
if batch_size > max_batch_size:
129+
raise ValueError(
130+
"Requested batch size is higher that the maximum allowed value."
131+
)
132+
133+
if batch_size < min_batch_size:
134+
raise ValueError(
135+
"Requested batch size is lower that the minimum allowed value."
123136
)
124137

125138
if warmup_ratio > 1 or warmup_ratio < 0:
@@ -346,9 +359,6 @@ def create(
346359
FinetuneResponse: Object containing information about fine-tuning job.
347360
"""
348361

349-
if model is None and from_checkpoint is None:
350-
raise ValueError("You must specify either a model or a checkpoint")
351-
352362
requestor = api_requestor.APIRequestor(
353363
client=self._client,
354364
)
@@ -737,9 +747,6 @@ async def create(
737747
FinetuneResponse: Object containing information about fine-tuning job.
738748
"""
739749

740-
if model is None and from_checkpoint is None:
741-
raise ValueError("You must specify either a model or a checkpoint")
742-
743750
requestor = api_requestor.APIRequestor(
744751
client=self._client,
745752
)

‎src/together/types/finetune.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,7 @@ class FinetuneRequest(BaseModel):
170170
# validation file id
171171
validation_file: str | None = None
172172
# base model string
173-
model: str
173+
model: str | None = None
174174
# number of epochs to train for
175175
n_epochs: int
176176
# training learning rate

‎tests/unit/test_finetune_resources.py

+263
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,263 @@
1+
import pytest
2+
3+
from together.resources.finetune import createFinetuneRequest
4+
from together.types.finetune import (
5+
FinetuneTrainingLimits,
6+
FinetuneFullTrainingLimits,
7+
FinetuneLoraTrainingLimits,
8+
)
9+
10+
11+
_MODEL_NAME = "meta-llama/Meta-Llama-3.1-8B-Instruct-Reference"
12+
_TRAINING_FILE = "file-7dbce5e9-7993-4520-9f3e-a7ece6c39d84"
13+
_VALIDATION_FILE = "file-7dbce5e9-7553-4520-9f3e-a7ece6c39d84"
14+
_FROM_CHECKPOINT = "ft-12345678-1234-1234-1234-1234567890ab"
15+
_MODEL_LIMITS = FinetuneTrainingLimits(
16+
max_num_epochs=20,
17+
max_learning_rate=1.0,
18+
min_learning_rate=1e-6,
19+
full_training=FinetuneFullTrainingLimits(
20+
max_batch_size=96,
21+
min_batch_size=8,
22+
),
23+
lora_training=FinetuneLoraTrainingLimits(
24+
max_batch_size=128,
25+
min_batch_size=8,
26+
max_rank=64,
27+
target_modules=["q", "k", "v", "o", "mlp"],
28+
),
29+
)
30+
31+
32+
def test_simple_request():
33+
request = createFinetuneRequest(
34+
model_limits=_MODEL_LIMITS,
35+
model=_MODEL_NAME,
36+
training_file=_TRAINING_FILE,
37+
)
38+
39+
assert request.model == _MODEL_NAME
40+
assert request.training_file == _TRAINING_FILE
41+
assert request.learning_rate > 0
42+
assert request.n_epochs > 0
43+
assert request.warmup_ratio == 0.0
44+
assert request.training_type.type == "Full"
45+
assert request.batch_size == _MODEL_LIMITS.full_training.max_batch_size
46+
47+
48+
def test_validation_file():
49+
request = createFinetuneRequest(
50+
model_limits=_MODEL_LIMITS,
51+
model=_MODEL_NAME,
52+
training_file=_TRAINING_FILE,
53+
validation_file=_VALIDATION_FILE,
54+
)
55+
56+
assert request.training_file == _TRAINING_FILE
57+
assert request.validation_file == _VALIDATION_FILE
58+
59+
60+
def test_no_training_file():
61+
with pytest.raises(
62+
TypeError, match="missing 1 required positional argument: 'training_file'"
63+
):
64+
_ = createFinetuneRequest(
65+
model_limits=_MODEL_LIMITS,
66+
model=_MODEL_NAME,
67+
)
68+
69+
70+
def test_lora_request():
71+
request = createFinetuneRequest(
72+
model_limits=_MODEL_LIMITS,
73+
model=_MODEL_NAME,
74+
training_file=_TRAINING_FILE,
75+
lora=True,
76+
)
77+
78+
assert request.training_type.type == "Lora"
79+
assert request.training_type.lora_r == _MODEL_LIMITS.lora_training.max_rank
80+
assert request.training_type.lora_alpha == _MODEL_LIMITS.lora_training.max_rank * 2
81+
assert request.training_type.lora_dropout == 0.0
82+
assert request.training_type.lora_trainable_modules == "all-linear"
83+
assert request.batch_size == _MODEL_LIMITS.lora_training.max_batch_size
84+
85+
86+
def test_from_checkpoint_request():
87+
request = createFinetuneRequest(
88+
model_limits=_MODEL_LIMITS,
89+
training_file=_TRAINING_FILE,
90+
from_checkpoint=_FROM_CHECKPOINT,
91+
)
92+
93+
assert request.model is None
94+
assert request.from_checkpoint == _FROM_CHECKPOINT
95+
96+
97+
def test_both_from_checkpoint_model_name():
98+
with pytest.raises(
99+
ValueError,
100+
match="You must specify either a model or a checkpoint to start a job from, not both",
101+
):
102+
_ = createFinetuneRequest(
103+
model_limits=_MODEL_LIMITS,
104+
model=_MODEL_NAME,
105+
training_file=_TRAINING_FILE,
106+
from_checkpoint=_FROM_CHECKPOINT,
107+
)
108+
109+
110+
def test_no_from_checkpoint_no_model_name():
111+
with pytest.raises(
112+
ValueError, match="You must specify either a model or a checkpoint"
113+
):
114+
_ = createFinetuneRequest(
115+
model_limits=_MODEL_LIMITS,
116+
training_file=_TRAINING_FILE,
117+
)
118+
119+
120+
def test_batch_size_limit():
121+
with pytest.raises(
122+
ValueError,
123+
match="Requested batch size is higher that the maximum allowed value",
124+
):
125+
_ = createFinetuneRequest(
126+
model_limits=_MODEL_LIMITS,
127+
model=_MODEL_NAME,
128+
training_file=_TRAINING_FILE,
129+
batch_size=128,
130+
)
131+
132+
with pytest.raises(
133+
ValueError, match="Requested batch size is lower that the minimum allowed value"
134+
):
135+
_ = createFinetuneRequest(
136+
model_limits=_MODEL_LIMITS,
137+
model=_MODEL_NAME,
138+
training_file=_TRAINING_FILE,
139+
batch_size=1,
140+
)
141+
142+
with pytest.raises(
143+
ValueError,
144+
match="Requested batch size is higher that the maximum allowed value",
145+
):
146+
_ = createFinetuneRequest(
147+
model_limits=_MODEL_LIMITS,
148+
model=_MODEL_NAME,
149+
training_file=_TRAINING_FILE,
150+
batch_size=256,
151+
lora=True,
152+
)
153+
154+
with pytest.raises(
155+
ValueError, match="Requested batch size is lower that the minimum allowed value"
156+
):
157+
_ = createFinetuneRequest(
158+
model_limits=_MODEL_LIMITS,
159+
model=_MODEL_NAME,
160+
training_file=_TRAINING_FILE,
161+
batch_size=1,
162+
lora=True,
163+
)
164+
165+
166+
def test_non_lora_model():
167+
with pytest.raises(
168+
ValueError, match="LoRA adapters are not supported for the selected model."
169+
):
170+
_ = createFinetuneRequest(
171+
model_limits=FinetuneTrainingLimits(
172+
max_num_epochs=20,
173+
max_learning_rate=1.0,
174+
min_learning_rate=1e-6,
175+
full_training=FinetuneFullTrainingLimits(
176+
max_batch_size=96,
177+
min_batch_size=8,
178+
),
179+
lora_training=None,
180+
),
181+
model=_MODEL_NAME,
182+
training_file=_TRAINING_FILE,
183+
lora=True,
184+
)
185+
186+
187+
def test_non_full_model():
188+
with pytest.raises(
189+
ValueError, match="Full training is not supported for the selected model."
190+
):
191+
_ = createFinetuneRequest(
192+
model_limits=FinetuneTrainingLimits(
193+
max_num_epochs=20,
194+
max_learning_rate=1.0,
195+
min_learning_rate=1e-6,
196+
lora_training=FinetuneLoraTrainingLimits(
197+
max_batch_size=96,
198+
min_batch_size=8,
199+
max_rank=64,
200+
target_modules=["q", "k", "v", "o", "mlp"],
201+
),
202+
full_training=None,
203+
),
204+
model=_MODEL_NAME,
205+
training_file=_TRAINING_FILE,
206+
lora=False,
207+
)
208+
209+
210+
@pytest.mark.parametrize("warmup_ratio", [-1.0, 2.0])
211+
def test_bad_warmup(warmup_ratio):
212+
with pytest.raises(ValueError, match="Warmup ratio should be between 0 and 1"):
213+
_ = createFinetuneRequest(
214+
model_limits=_MODEL_LIMITS,
215+
model=_MODEL_NAME,
216+
training_file=_TRAINING_FILE,
217+
warmup_ratio=warmup_ratio,
218+
)
219+
220+
221+
@pytest.mark.parametrize("min_lr_ratio", [-1.0, 2.0])
222+
def test_bad_min_lr_ratio(min_lr_ratio):
223+
with pytest.raises(
224+
ValueError, match="Min learning rate ratio should be between 0 and 1"
225+
):
226+
_ = createFinetuneRequest(
227+
model_limits=_MODEL_LIMITS,
228+
model=_MODEL_NAME,
229+
training_file=_TRAINING_FILE,
230+
min_lr_ratio=min_lr_ratio,
231+
)
232+
233+
234+
@pytest.mark.parametrize("max_grad_norm", [-1.0, -0.01])
235+
def test_bad_max_grad_norm(max_grad_norm):
236+
with pytest.raises(ValueError, match="Max gradient norm should be non-negative"):
237+
_ = createFinetuneRequest(
238+
model_limits=_MODEL_LIMITS,
239+
model=_MODEL_NAME,
240+
training_file=_TRAINING_FILE,
241+
max_grad_norm=max_grad_norm,
242+
)
243+
244+
245+
@pytest.mark.parametrize("weight_decay", [-1.0, -0.01])
246+
def test_bad_weight_decay(weight_decay):
247+
with pytest.raises(ValueError, match="Weight decay should be non-negative"):
248+
_ = createFinetuneRequest(
249+
model_limits=_MODEL_LIMITS,
250+
model=_MODEL_NAME,
251+
training_file=_TRAINING_FILE,
252+
weight_decay=weight_decay,
253+
)
254+
255+
256+
def test_bad_training_method():
257+
with pytest.raises(ValueError, match="training_method must be one of .*"):
258+
_ = createFinetuneRequest(
259+
model_limits=_MODEL_LIMITS,
260+
model=_MODEL_NAME,
261+
training_file=_TRAINING_FILE,
262+
training_method="NON_SFT",
263+
)

0 commit comments

Comments
 (0)