ENG 1730: Support validation files for fine-tuning jobs (#161)

* Add n_evals and validation_file

* Add eval complete, validation file arg

* Add eval complete, validation file arg

* Change to "File ID"

Co-authored-by: Max Ryabinin <[email protected]>

* Change to "File ID" 2

Co-authored-by: Max Ryabinin <[email protected]>

* Change to log warn

* Update pyproject version

* Undo lock commit

---------

Co-authored-by: Arsh Zahed <[email protected]>
Co-authored-by: Max Ryabinin <[email protected]>

Author: 3 people

src/together/cli/api/finetune.py

@@ -23,6 +23,10 @@ def fine_tuning(ctx: click.Context) -> None:
 )
 @click.option("--model", type=str, required=True, help="Base model name")
 @click.option("--n-epochs", type=int, default=1, help="Number of epochs to train for")
+@click.option(
+    "--validation-file", type=str, default="", help="Validation file ID from Files API"
+)
+@click.option("--n-evals", type=int, default=0, help="Number of evaluation loops")
 @click.option(
     "--n-checkpoints", type=int, default=1, help="Number of checkpoints to save"
 )
@@ -50,8 +54,10 @@ def fine_tuning(ctx: click.Context) -> None:
 def create(
     ctx: click.Context,
     training_file: str,
+    validation_file: str,
     model: str,
     n_epochs: int,
+    n_evals: int,
     n_checkpoints: int,
     batch_size: int,
     learning_rate: float,
@@ -80,11 +86,21 @@ def create(
                     f"You set LoRA parameter `{param}` for a full fine-tuning job. "
                     f"Please change the job type with --lora or remove `{param}` from the arguments"
                 )
+    if n_evals <= 0 and validation_file:
+        log_warn(
+            "Warning: You have specified a validation file but the number of evaluation loops is set to 0. No evaluations will be performed."
+        )
+    elif n_evals > 0 and not validation_file:
+        raise click.BadParameter(
+            "You have specified a number of evaluation loops but no validation file."
+        )
 
     response = client.fine_tuning.create(
         training_file=training_file,
         model=model,
         n_epochs=n_epochs,
+        validation_file=validation_file,
+        n_evals=n_evals,
         n_checkpoints=n_checkpoints,
         batch_size=batch_size,
         learning_rate=learning_rate,

src/together/resources/finetune.py

@@ -30,6 +30,8 @@ def create(
         training_file: str,
         model: str,
         n_epochs: int = 1,
+        validation_file: str | None = "",
+        n_evals: int | None = 0,
         n_checkpoints: int | None = 1,
         batch_size: int | None = 16,
         learning_rate: float | None = 0.00001,
@@ -48,6 +50,8 @@ def create(
             training_file (str): File-ID of a file uploaded to the Together API
             model (str): Name of the base model to run fine-tune job on
             n_epochs (int, optional): Number of epochs for fine-tuning. Defaults to 1.
+            validation file (str, optional): File ID of a file uploaded to the Together API for validation.
+            n_evals (int, optional): Number of evaluation loops to run. Defaults to 0.
             n_checkpoints (int, optional): Number of checkpoints to save during fine-tuning.
                 Defaults to 1.
             batch_size (int, optional): Batch size for fine-tuning. Defaults to 32.
@@ -83,7 +87,9 @@ def create(
         parameter_payload = FinetuneRequest(
             model=model,
             training_file=training_file,
+            validation_file=validation_file,
             n_epochs=n_epochs,
+            n_evals=n_evals,
             n_checkpoints=n_checkpoints,
             batch_size=batch_size,
             learning_rate=learning_rate,
@@ -275,6 +281,8 @@ async def create(
         training_file: str,
         model: str,
         n_epochs: int = 1,
+        validation_file: str | None = "",
+        n_evals: int = 0,
         n_checkpoints: int | None = 1,
         batch_size: int | None = 32,
         learning_rate: float = 0.00001,
@@ -288,6 +296,8 @@ async def create(
             training_file (str): File-ID of a file uploaded to the Together API
             model (str): Name of the base model to run fine-tune job on
             n_epochs (int, optional): Number of epochs for fine-tuning. Defaults to 1.
+            validation file (str, optional): File ID of a file uploaded to the Together API for validation.
+            n_evals (int, optional): Number of evaluation loops to run. Defaults to 0.
             n_checkpoints (int, optional): Number of checkpoints to save during fine-tuning.
                 Defaults to 1.
             batch_size (int, optional): Batch size for fine-tuning. Defaults to 32.
@@ -309,7 +319,9 @@ async def create(
         parameter_payload = FinetuneRequest(
             model=model,
             training_file=training_file,
+            validation_file=validation_file,
             n_epochs=n_epochs,
+            n_evals=n_evals,
             n_checkpoints=n_checkpoints,
             batch_size=batch_size,
             learning_rate=learning_rate,

src/together/types/finetune.py

@@ -61,6 +61,7 @@ class FinetuneEventType(str, Enum):
     CHECKPOINT_SAVE = "CHECKPOINT_SAVE"
     BILLING_LIMIT = "BILLING_LIMIT"
     EPOCH_COMPLETE = "EPOCH_COMPLETE"
+    EVAL_COMPLETE = "EVAL_COMPLETE"
     TRAINING_COMPLETE = "TRAINING_COMPLETE"
     MODEL_COMPRESSING = "COMPRESSING_MODEL"
     MODEL_COMPRESSION_COMPLETE = "MODEL_COMPRESSION_COMPLETE"
@@ -135,6 +136,8 @@ class FinetuneRequest(BaseModel):
 
     # training file ID
     training_file: str
+    # validation file id
+    validation_file: str | None = None
     # base model string
     model: str
     # number of epochs to train for
@@ -143,6 +146,8 @@ class FinetuneRequest(BaseModel):
     learning_rate: float
     # number of checkpoints to save
     n_checkpoints: int | None = None
+    # number of evaluation loops to run
+    n_evals: int | None = None
     # training batch size
     batch_size: int | None = None
     # up to 40 character suffix for output model name
@@ -173,6 +178,8 @@ class FinetuneResponse(BaseModel):
     n_epochs: int | None = None
     # number of checkpoints to save
     n_checkpoints: int | None = None
+    # number of evaluation loops
+    n_evals: int | None = None
     # training batch size
     batch_size: int | None = None
     # training learning rate
@@ -196,8 +203,14 @@ class FinetuneResponse(BaseModel):
     param_count: int | None = None
     # fine-tune job price
     total_price: int | None = None
+    # total number of training steps
+    total_steps: int | None = None
+    # number of steps completed (incrementing counter)
+    steps_completed: int | None = None
     # number of epochs completed (incrementing counter)
     epochs_completed: int | None = None
+    # number of evaluation loops completed (incrementing counter)
+    evals_completed: int | None = None
     # place in job queue (decrementing counter)
     queue_depth: int | None = None
     # weights & biases project name