diff --git a/docs/customizer/tutorials/sft-customization-job.ipynb b/docs/customizer/tutorials/sft-customization-job.ipynb
index 91d3c25899..a7a0146b32 100644
--- a/docs/customizer/tutorials/sft-customization-job.ipynb
+++ b/docs/customizer/tutorials/sft-customization-job.ipynb
@@ -79,7 +79,9 @@
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {},
+ "outputs": [],
"source": [
"import json\n",
"import os\n",
@@ -90,9 +92,7 @@
" base_url=NMP_BASE_URL,\n",
" workspace=\"default\"\n",
")"
- ],
- "execution_count": null,
- "outputs": []
+ ]
},
{
"cell_type": "markdown",
@@ -201,7 +201,9 @@
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {},
+ "outputs": [],
"source": [
"from pathlib import Path\n",
"from datasets import load_dataset, DatasetDict\n",
@@ -265,13 +267,13 @@
" sample = json.loads(first_line)\n",
" print(f\"Prompt: {sample['prompt'][:200]}...\")\n",
" print(f\"Completion: {sample['completion']}\")"
- ],
- "execution_count": null,
- "outputs": []
+ ]
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {},
+ "outputs": [],
"source": [
"# Create fileset to store SFT training data\n",
"DATASET_NAME = \"sft-dataset\"\n",
@@ -297,9 +299,7 @@
"# Validate training data is uploaded correctly\n",
"print(\"Training data:\")\n",
"print(json.dumps([f.model_dump() for f in client.files.list(fileset=DATASET_NAME, workspace=\"default\").data], indent=2))"
- ],
- "execution_count": null,
- "outputs": []
+ ]
},
{
"cell_type": "markdown",
@@ -335,7 +335,9 @@
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {},
+ "outputs": [],
"source": [
"# Export the HF_TOKEN and NGC_API_KEY environment variables if they are not already set.\n",
"# This tutorial's default model (meta-llama/Llama-3.2-1B-Instruct) is gated and requires HF_TOKEN.\n",
@@ -370,9 +372,7 @@
"# Create NGC API key secret\n",
"# Uncomment the line below if you have NGC API Key and want to finetune NGC models\n",
"# ngc_api_key = create_or_get_secret(\"ngc-api-key\", NGC_API_KEY, \"NGC_API_KEY\")"
- ],
- "execution_count": null,
- "outputs": []
+ ]
},
{
"cell_type": "markdown",
@@ -387,7 +387,9 @@
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {},
+ "outputs": [],
"source": [
"import time\n",
"\n",
@@ -454,9 +456,7 @@
" )\n",
"\n",
"print(f\"ModelSpec populated: {base_model.spec}\")"
- ],
- "execution_count": null,
- "outputs": []
+ ]
},
{
"cell_type": "markdown",
@@ -482,7 +482,9 @@
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {},
+ "outputs": [],
"source": [
"import uuid\n",
"from nemo_automodel_plugin.schema import AutomodelJobInput\n",
@@ -518,9 +520,7 @@
"\n",
"print(f\"Submitted job: {job.job.name}\")\n",
"print(f\"Output model: {OUTPUT_NAME}\")\n"
- ],
- "execution_count": null,
- "outputs": []
+ ]
},
{
"cell_type": "markdown",
@@ -531,7 +531,9 @@
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {},
+ "outputs": [],
"source": [
"import time\n",
"from IPython.display import clear_output\n",
@@ -578,9 +580,7 @@
"\n",
"if status.status != \"completed\":\n",
" raise RuntimeError(f\"Training job finished with status: {status.status}\")"
- ],
- "execution_count": null,
- "outputs": []
+ ]
},
{
"cell_type": "markdown",
@@ -611,18 +611,20 @@
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {},
+ "outputs": [],
"source": [
"# Validate model entity exists\n",
"model_entity = client.models.retrieve(workspace='default', name=OUTPUT_NAME)\n",
"print(model_entity.model_dump_json(indent=2))"
- ],
- "execution_count": null,
- "outputs": []
+ ]
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {},
+ "outputs": [],
"source": [
"# Create deployment config\n",
"deploy_suffix = uuid.uuid4().hex[:4]\n",
@@ -660,9 +662,7 @@
"\n",
"print(f\"Deployment name: {deployment.name}\")\n",
"print(f\"Deployment status: {deployment_status.status}\")\n"
- ],
- "execution_count": null,
- "outputs": []
+ ]
},
{
"cell_type": "markdown",
@@ -706,7 +706,9 @@
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {},
+ "outputs": [],
"source": [
"import time\n",
"from IPython.display import clear_output\n",
@@ -750,9 +752,7 @@
" raise TimeoutError(f\"Deployment timeout after {TIMEOUT_MINUTES} minutes\")\n",
"\n",
" time.sleep(15)"
- ],
- "execution_count": null,
- "outputs": []
+ ]
},
{
"cell_type": "markdown",
@@ -767,7 +767,9 @@
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {},
+ "outputs": [],
"source": [
"# Wait for deployment to be ready, then test\n",
"# Test the fine-tuned model with a question answering prompt\n",
@@ -794,9 +796,7 @@
"print(f\"Question: {question}\")\n",
"print(f\"Expected: Neil Armstrong\")\n",
"print(f\"Model output: {response['choices'][0]['text']}\")"
- ],
- "execution_count": null,
- "outputs": []
+ ]
},
{
"cell_type": "markdown",
@@ -863,7 +863,8 @@
"\n",
"**Deployment fails:**\n",
"- Verify output model exists: `client.models.retrieve(name=OUTPUT_NAME, workspace=\"default\")`\n",
- "- Check deployment logs: `client.inference.deployments.get_logs(name=deployment.name, workspace=\"default\")`\n",
+ "- Check job logs: `client.jobs.get_logs(name=job.job.name, workspace=\"default\")`\n",
+ "- Check deployment status: `client.inference.deployments.retrieve(name=deployment.name, workspace=\"default\")`\n",
"- Ensure sufficient GPU resources for `executor_config={\"gpu\": 1, ...}`\n",
"- Verify the deployment config matches this tutorial: `engine=\"vllm\"` with `vllm/vllm-openai:v0.22.1`\n",
"\n",
@@ -889,4 +890,4 @@
},
"nbformat": 4,
"nbformat_minor": 2
-}
\ No newline at end of file
+}
diff --git a/docs/customizer/tutorials/sft-customization-job.mdx b/docs/customizer/tutorials/sft-customization-job.mdx
index 60b3928efa..8be12692b3 100644
--- a/docs/customizer/tutorials/sft-customization-job.mdx
+++ b/docs/customizer/tutorials/sft-customization-job.mdx
@@ -1,7 +1,6 @@
---
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
-
title: "Full SFT Customization"
description: ""
---
@@ -231,12 +230,14 @@ If you plan to use NGC or Hugging Face models, you will need to configure authen
- **NGC models** (`ngc://` URIs): Requires NGC API key
- **Hugging Face models** (`hf://` URIs): Requires HF token for gated/private models
+
Configure these as secrets in your platform. Refer to [Managing Secrets](/documentation/get-started/core-concepts/manage-secrets) for detailed instructions.
Get your credentials to access base models:
- [NGC API Key](https://ngc.nvidia.com/) (Setup → Generate API Key)
- [Hugging Face Token](https://huggingface.co/settings/tokens) (Create token with Read access)
+
---
#### Quick Setup Example
@@ -260,6 +261,7 @@ if not HF_TOKEN:
"The default model meta-llama/Llama-3.2-1B-Instruct is gated."
)
+
def create_or_get_secret(name: str, value: str, label: str):
try:
secret = client.secrets.create(
@@ -273,6 +275,7 @@ def create_or_get_secret(name: str, value: str, label: str):
print(f"Secret '{name}' already exists, continuing...")
return client.secrets.retrieve(name=name, workspace="default")
+
# Create Hugging Face token secret
hf_secret = create_or_get_secret("hf-token", HF_TOKEN, "HF_TOKEN")
print("HF_TOKEN secret:")
@@ -507,6 +510,7 @@ deployment = client.inference.deployments.create(
config=deployment_config.name
)
+
# Check deployment status
deployment_status = client.inference.deployments.retrieve(
name=deployment.name,
@@ -651,6 +655,7 @@ For detailed information on all available hyperparameters, recommended values, a
---
+
## Troubleshooting
**Job fails during model download:**
@@ -687,10 +692,12 @@ For detailed information on all available hyperparameters, recommended values, a
**Deployment fails:**
- Verify output model exists: `client.models.retrieve(name=OUTPUT_NAME, workspace="default")`
-- Check deployment logs: `client.inference.deployments.get_logs(name=deployment.name, workspace="default")`
+- Check job logs: `client.jobs.get_logs(name=job.job.name, workspace="default")`
+- Check deployment status: `client.inference.deployments.retrieve(name=deployment.name, workspace="default")`
- Ensure sufficient GPU resources for `executor_config={"gpu": 1, ...}`
- Verify the deployment config matches this tutorial: `engine="vllm"` with `vllm/vllm-openai:v0.22.1`
+
## Next Steps
- [Monitor training metrics](/documentation/customizer-reference/tutorials/metrics) in detail
diff --git a/docs/fern/components/notebooks/distillation-customization-job.json b/docs/fern/components/notebooks/distillation-customization-job.json
index 3ae0a8e082..01a2b891b7 100644
--- a/docs/fern/components/notebooks/distillation-customization-job.json
+++ b/docs/fern/components/notebooks/distillation-customization-job.json
@@ -196,8 +196,8 @@
},
{
"type": "markdown",
- "source": "**Interpreting ROUGE Scores:**\n\n| Metric | Measures |\n|--------|----------|\n| **ROUGE-1** | Unigram overlap between prediction and reference |\n| **ROUGE-2** | Bigram overlap (captures phrase-level similarity) |\n| **ROUGE-L** | Longest common subsequence (captures sentence structure) |\n| **ROUGE-Lsum** | ROUGE-L computed over full summaries |\n\n**What to expect:**\n- The base student (1B, no training) provides a lower bound since it has not seen the task data\n- The distilled student (1B, KD) should significantly outperform the base student, demonstrating the knowledge transferred from the 3B teacher\n- If the distilled student scores are not much higher than the baseline, try increasing `distillation_temperature`, adjusting `distillation_ratio`, or training for more epochs\n\n---\n\n## Hyperparameters\n\nFor detailed information on all available hyperparameters, recommended values, and tuning guidance, refer to the [Hyperparameter Reference](../manage-customization-jobs/hyperparameters.md).\n\n---\n\n## Troubleshooting\n\n**Job fails during model download:**\n- Verify authentication secrets are configured (refer to [Managing Secrets](../../get-started/concepts/manage-secrets.md))\n- For gated Hugging Face models (Llama, Gemma), accept the license on the model page\n- Check both `model` (student) and `teacher_model` URNs are correct\n- Ensure both model entities exist: `client.models.retrieve(name=..., workspace=\"default\")`\n\n**Job fails with OOM (Out of Memory) error:**\n\nKD loads both models, so OOM is more likely than with SFT:\n1. **First try:** Use `teacher_precision=\"bf16\"` to reduce teacher memory\n2. **Still OOM:** Reduce `micro_batch_size` to 1\n3. **Still OOM:** Reduce `global_batch_size` and `max_seq_length`\n4. **Last resort:** Increase `num_gpus_per_node`\n\n**No chat template / `/chat/completions` fails:**\n- Use Instruct model variants (e.g., `Llama-3.2-1B-Instruct`) instead of base models (`Llama-3.2-1B`). Base models do not include a chat template in their tokenizer, so the output model will also lack one.\n\n**Distilled model quality is poor:**\n- Increase `distillation_temperature` (try 2.0–5.0) to transfer more nuanced knowledge\n- Adjust `distillation_ratio`—if dataset labels are high-quality, lower the ratio; if the teacher is strong, raise it\n- Increase `epochs` or `max_steps` for more training\n- Verify teacher and student share the same vocabulary\n\n**Vocabulary mismatch error:**\n- Teacher and student must use the same tokenizer. Use models from the same family (e.g., Llama 3.2 1B Instruct + Llama 3.2 3B Instruct)\n\n**Deployment fails:**\n- Verify output model exists: `client.models.retrieve(name=DISTILLED_STUDENT_NAME, workspace=\"default\")`\n- Check deployment logs: `client.inference.deployments.get_logs(name=deployment.name, workspace=\"default\")`\n- The distilled model has the same size as the student, so GPU requirements match the student model\n\n\n## Next Steps\n\n- [Monitor training metrics](fine-tune-metrics) in detail\n- [Evaluate your fine-tuned model](../../evaluator/index) using the Evaluator service\n- Learn about [LoRA customization](./lora-customization-job) for resource-efficient fine-tuning\n- Learn about [Full SFT](./sft-customization-job) for direct supervised fine-tuning",
- "source_html": "Interpreting ROUGE Scores:
\n\n\n\n| Metric | \nMeasures | \n
\n\n\n\n| ROUGE-1 | \nUnigram overlap between prediction and reference | \n
\n\n| ROUGE-2 | \nBigram overlap (captures phrase-level similarity) | \n
\n\n| ROUGE-L | \nLongest common subsequence (captures sentence structure) | \n
\n\n| ROUGE-Lsum | \nROUGE-L computed over full summaries | \n
\n\n
\nWhat to expect:
\n\n- The base student (1B, no training) provides a lower bound since it has not seen the task data
\n- The distilled student (1B, KD) should significantly outperform the base student, demonstrating the knowledge transferred from the 3B teacher
\n- If the distilled student scores are not much higher than the baseline, try increasing
distillation_temperature, adjusting distillation_ratio, or training for more epochs \n
\n
\nHyperparameters
\nFor detailed information on all available hyperparameters, recommended values, and tuning guidance, refer to the Hyperparameter Reference.
\n
\nTroubleshooting
\nJob fails during model download:
\n\n- Verify authentication secrets are configured (refer to Managing Secrets)
\n- For gated Hugging Face models (Llama, Gemma), accept the license on the model page
\n- Check both
model (student) and teacher_model URNs are correct \n- Ensure both model entities exist:
client.models.retrieve(name=..., workspace="default") \n
\nJob fails with OOM (Out of Memory) error:
\nKD loads both models, so OOM is more likely than with SFT:
\n\n- First try: Use
teacher_precision="bf16" to reduce teacher memory \n- Still OOM: Reduce
micro_batch_size to 1 \n- Still OOM: Reduce
global_batch_size and max_seq_length \n- Last resort: Increase
num_gpus_per_node \n
\nNo chat template / /chat/completions fails:
\n\n- Use Instruct model variants (e.g.,
Llama-3.2-1B-Instruct) instead of base models (Llama-3.2-1B). Base models do not include a chat template in their tokenizer, so the output model will also lack one. \n
\nDistilled model quality is poor:
\n\n- Increase
distillation_temperature (try 2.0–5.0) to transfer more nuanced knowledge \n- Adjust
distillation_ratio—if dataset labels are high-quality, lower the ratio; if the teacher is strong, raise it \n- Increase
epochs or max_steps for more training \n- Verify teacher and student share the same vocabulary
\n
\nVocabulary mismatch error:
\n\n- Teacher and student must use the same tokenizer. Use models from the same family (e.g., Llama 3.2 1B Instruct + Llama 3.2 3B Instruct)
\n
\nDeployment fails:
\n\n- Verify output model exists:
client.models.retrieve(name=DISTILLED_STUDENT_NAME, workspace="default") \n- Check deployment logs:
client.inference.deployments.get_logs(name=deployment.name, workspace="default") \n- The distilled model has the same size as the student, so GPU requirements match the student model
\n
\nNext Steps
\n\n"
+ "source": "**Interpreting ROUGE Scores:**\n\n| Metric | Measures |\n|--------|----------|\n| **ROUGE-1** | Unigram overlap between prediction and reference |\n| **ROUGE-2** | Bigram overlap (captures phrase-level similarity) |\n| **ROUGE-L** | Longest common subsequence (captures sentence structure) |\n| **ROUGE-Lsum** | ROUGE-L computed over full summaries |\n\n**What to expect:**\n- The base student (1B, no training) provides a lower bound since it has not seen the task data\n- The distilled student (1B, KD) should significantly outperform the base student, demonstrating the knowledge transferred from the 3B teacher\n- If the distilled student scores are not much higher than the baseline, try increasing `distillation_temperature`, adjusting `distillation_ratio`, or training for more epochs\n\n---\n\n## Hyperparameters\n\nFor detailed information on all available hyperparameters, recommended values, and tuning guidance, refer to the [Hyperparameter Reference](../manage-customization-jobs/hyperparameters.md).\n\n---\n\n## Troubleshooting\n\n**Job fails during model download:**\n- Verify authentication secrets are configured (refer to [Managing Secrets](../../get-started/concepts/manage-secrets.md))\n- For gated Hugging Face models (Llama, Gemma), accept the license on the model page\n- Check both `model` (student) and `teacher_model` URNs are correct\n- Ensure both model entities exist: `client.models.retrieve(name=..., workspace=\"default\")`\n\n**Job fails with OOM (Out of Memory) error:**\n\nKD loads both models, so OOM is more likely than with SFT:\n1. **First try:** Use `teacher_precision=\"bf16\"` to reduce teacher memory\n2. **Still OOM:** Reduce `micro_batch_size` to 1\n3. **Still OOM:** Reduce `global_batch_size` and `max_seq_length`\n4. **Last resort:** Increase `num_gpus_per_node`\n\n**No chat template / `/chat/completions` fails:**\n- Use Instruct model variants (e.g., `Llama-3.2-1B-Instruct`) instead of base models (`Llama-3.2-1B`). Base models do not include a chat template in their tokenizer, so the output model will also lack one.\n\n**Distilled model quality is poor:**\n- Increase `distillation_temperature` (try 2.0–5.0) to transfer more nuanced knowledge\n- Adjust `distillation_ratio`—if dataset labels are high-quality, lower the ratio; if the teacher is strong, raise it\n- Increase `epochs` or `max_steps` for more training\n- Verify teacher and student share the same vocabulary\n\n**Vocabulary mismatch error:**\n- Teacher and student must use the same tokenizer. Use models from the same family (e.g., Llama 3.2 1B Instruct + Llama 3.2 3B Instruct)\n\n**Deployment fails:**\n- Verify output model exists: `client.models.retrieve(name=DISTILLED_STUDENT_NAME, workspace=\"default\")`\n- Check job logs: `client.jobs.get_logs(name=kd_job.job.name, workspace=\"default\")`\n- Check student deployment status: `client.inference.deployments.retrieve(name=student_deployment.name, workspace=\"default\")`\n- The distilled model has the same size as the student, so GPU requirements match the student model\n\n\n## Next Steps\n\n- [Monitor training metrics](fine-tune-metrics) in detail\n- [Evaluate your fine-tuned model](../../evaluator/index) using the Evaluator service\n- Learn about [LoRA customization](./lora-customization-job) for resource-efficient fine-tuning\n- Learn about [Full SFT](./sft-customization-job) for direct supervised fine-tuning",
+ "source_html": "Interpreting ROUGE Scores:
\n\n\n\n| Metric | \nMeasures | \n
\n\n\n\n| ROUGE-1 | \nUnigram overlap between prediction and reference | \n
\n\n| ROUGE-2 | \nBigram overlap (captures phrase-level similarity) | \n
\n\n| ROUGE-L | \nLongest common subsequence (captures sentence structure) | \n
\n\n| ROUGE-Lsum | \nROUGE-L computed over full summaries | \n
\n\n
\nWhat to expect:
\n\n- The base student (1B, no training) provides a lower bound since it has not seen the task data
\n- The distilled student (1B, KD) should significantly outperform the base student, demonstrating the knowledge transferred from the 3B teacher
\n- If the distilled student scores are not much higher than the baseline, try increasing
distillation_temperature, adjusting distillation_ratio, or training for more epochs \n
\n
\nHyperparameters
\nFor detailed information on all available hyperparameters, recommended values, and tuning guidance, refer to the Hyperparameter Reference.
\n
\nTroubleshooting
\nJob fails during model download:
\n\n- Verify authentication secrets are configured (refer to Managing Secrets)
\n- For gated Hugging Face models (Llama, Gemma), accept the license on the model page
\n- Check both
model (student) and teacher_model URNs are correct \n- Ensure both model entities exist:
client.models.retrieve(name=..., workspace="default") \n
\nJob fails with OOM (Out of Memory) error:
\nKD loads both models, so OOM is more likely than with SFT:
\n\n- First try: Use
teacher_precision="bf16" to reduce teacher memory \n- Still OOM: Reduce
micro_batch_size to 1 \n- Still OOM: Reduce
global_batch_size and max_seq_length \n- Last resort: Increase
num_gpus_per_node \n
\nNo chat template / /chat/completions fails:
\n\n- Use Instruct model variants (e.g.,
Llama-3.2-1B-Instruct) instead of base models (Llama-3.2-1B). Base models do not include a chat template in their tokenizer, so the output model will also lack one. \n
\nDistilled model quality is poor:
\n\n- Increase
distillation_temperature (try 2.0–5.0) to transfer more nuanced knowledge \n- Adjust
distillation_ratio—if dataset labels are high-quality, lower the ratio; if the teacher is strong, raise it \n- Increase
epochs or max_steps for more training \n- Verify teacher and student share the same vocabulary
\n
\nVocabulary mismatch error:
\n\n- Teacher and student must use the same tokenizer. Use models from the same family (e.g., Llama 3.2 1B Instruct + Llama 3.2 3B Instruct)
\n
\nDeployment fails:
\n\n- Verify output model exists:
client.models.retrieve(name=DISTILLED_STUDENT_NAME, workspace="default") \n- Check job logs:
client.jobs.get_logs(name=kd_job.job.name, workspace="default") \n- Check student deployment status:
client.inference.deployments.retrieve(name=student_deployment.name, workspace="default") \n- The distilled model has the same size as the student, so GPU requirements match the student model
\n
\nNext Steps
\n\n"
}
]
}
\ No newline at end of file
diff --git a/docs/fern/components/notebooks/distillation-customization-job.ts b/docs/fern/components/notebooks/distillation-customization-job.ts
index 83d6f3a29e..1e75137fbb 100644
--- a/docs/fern/components/notebooks/distillation-customization-job.ts
+++ b/docs/fern/components/notebooks/distillation-customization-job.ts
@@ -201,7 +201,7 @@ export default { cells: [
},
{
"type": "markdown",
- "source": "**Interpreting ROUGE Scores:**\n\n| Metric | Measures |\n|--------|----------|\n| **ROUGE-1** | Unigram overlap between prediction and reference |\n| **ROUGE-2** | Bigram overlap (captures phrase-level similarity) |\n| **ROUGE-L** | Longest common subsequence (captures sentence structure) |\n| **ROUGE-Lsum** | ROUGE-L computed over full summaries |\n\n**What to expect:**\n- The base student (1B, no training) provides a lower bound since it has not seen the task data\n- The distilled student (1B, KD) should significantly outperform the base student, demonstrating the knowledge transferred from the 3B teacher\n- If the distilled student scores are not much higher than the baseline, try increasing `distillation_temperature`, adjusting `distillation_ratio`, or training for more epochs\n\n---\n\n## Hyperparameters\n\nFor detailed information on all available hyperparameters, recommended values, and tuning guidance, refer to the [Hyperparameter Reference](../manage-customization-jobs/hyperparameters.md).\n\n---\n\n## Troubleshooting\n\n**Job fails during model download:**\n- Verify authentication secrets are configured (refer to [Managing Secrets](../../get-started/concepts/manage-secrets.md))\n- For gated Hugging Face models (Llama, Gemma), accept the license on the model page\n- Check both `model` (student) and `teacher_model` URNs are correct\n- Ensure both model entities exist: `client.models.retrieve(name=..., workspace=\"default\")`\n\n**Job fails with OOM (Out of Memory) error:**\n\nKD loads both models, so OOM is more likely than with SFT:\n1. **First try:** Use `teacher_precision=\"bf16\"` to reduce teacher memory\n2. **Still OOM:** Reduce `micro_batch_size` to 1\n3. **Still OOM:** Reduce `global_batch_size` and `max_seq_length`\n4. **Last resort:** Increase `num_gpus_per_node`\n\n**No chat template / `/chat/completions` fails:**\n- Use Instruct model variants (e.g., `Llama-3.2-1B-Instruct`) instead of base models (`Llama-3.2-1B`). Base models do not include a chat template in their tokenizer, so the output model will also lack one.\n\n**Distilled model quality is poor:**\n- Increase `distillation_temperature` (try 2.0–5.0) to transfer more nuanced knowledge\n- Adjust `distillation_ratio`—if dataset labels are high-quality, lower the ratio; if the teacher is strong, raise it\n- Increase `epochs` or `max_steps` for more training\n- Verify teacher and student share the same vocabulary\n\n**Vocabulary mismatch error:**\n- Teacher and student must use the same tokenizer. Use models from the same family (e.g., Llama 3.2 1B Instruct + Llama 3.2 3B Instruct)\n\n**Deployment fails:**\n- Verify output model exists: `client.models.retrieve(name=DISTILLED_STUDENT_NAME, workspace=\"default\")`\n- Check deployment logs: `client.inference.deployments.get_logs(name=deployment.name, workspace=\"default\")`\n- The distilled model has the same size as the student, so GPU requirements match the student model\n\n\n## Next Steps\n\n- [Monitor training metrics](fine-tune-metrics) in detail\n- [Evaluate your fine-tuned model](../../evaluator/index) using the Evaluator service\n- Learn about [LoRA customization](./lora-customization-job) for resource-efficient fine-tuning\n- Learn about [Full SFT](./sft-customization-job) for direct supervised fine-tuning",
- "source_html": "Interpreting ROUGE Scores:
\n\n\n\n| Metric | \nMeasures | \n
\n\n\n\n| ROUGE-1 | \nUnigram overlap between prediction and reference | \n
\n\n| ROUGE-2 | \nBigram overlap (captures phrase-level similarity) | \n
\n\n| ROUGE-L | \nLongest common subsequence (captures sentence structure) | \n
\n\n| ROUGE-Lsum | \nROUGE-L computed over full summaries | \n
\n\n
\nWhat to expect:
\n\n- The base student (1B, no training) provides a lower bound since it has not seen the task data
\n- The distilled student (1B, KD) should significantly outperform the base student, demonstrating the knowledge transferred from the 3B teacher
\n- If the distilled student scores are not much higher than the baseline, try increasing
distillation_temperature, adjusting distillation_ratio, or training for more epochs \n
\n
\nHyperparameters
\nFor detailed information on all available hyperparameters, recommended values, and tuning guidance, refer to the Hyperparameter Reference.
\n
\nTroubleshooting
\nJob fails during model download:
\n\n- Verify authentication secrets are configured (refer to Managing Secrets)
\n- For gated Hugging Face models (Llama, Gemma), accept the license on the model page
\n- Check both
model (student) and teacher_model URNs are correct \n- Ensure both model entities exist:
client.models.retrieve(name=..., workspace="default") \n
\nJob fails with OOM (Out of Memory) error:
\nKD loads both models, so OOM is more likely than with SFT:
\n\n- First try: Use
teacher_precision="bf16" to reduce teacher memory \n- Still OOM: Reduce
micro_batch_size to 1 \n- Still OOM: Reduce
global_batch_size and max_seq_length \n- Last resort: Increase
num_gpus_per_node \n
\nNo chat template / /chat/completions fails:
\n\n- Use Instruct model variants (e.g.,
Llama-3.2-1B-Instruct) instead of base models (Llama-3.2-1B). Base models do not include a chat template in their tokenizer, so the output model will also lack one. \n
\nDistilled model quality is poor:
\n\n- Increase
distillation_temperature (try 2.0–5.0) to transfer more nuanced knowledge \n- Adjust
distillation_ratio—if dataset labels are high-quality, lower the ratio; if the teacher is strong, raise it \n- Increase
epochs or max_steps for more training \n- Verify teacher and student share the same vocabulary
\n
\nVocabulary mismatch error:
\n\n- Teacher and student must use the same tokenizer. Use models from the same family (e.g., Llama 3.2 1B Instruct + Llama 3.2 3B Instruct)
\n
\nDeployment fails:
\n\n- Verify output model exists:
client.models.retrieve(name=DISTILLED_STUDENT_NAME, workspace="default") \n- Check deployment logs:
client.inference.deployments.get_logs(name=deployment.name, workspace="default") \n- The distilled model has the same size as the student, so GPU requirements match the student model
\n
\nNext Steps
\n\n"
+ "source": "**Interpreting ROUGE Scores:**\n\n| Metric | Measures |\n|--------|----------|\n| **ROUGE-1** | Unigram overlap between prediction and reference |\n| **ROUGE-2** | Bigram overlap (captures phrase-level similarity) |\n| **ROUGE-L** | Longest common subsequence (captures sentence structure) |\n| **ROUGE-Lsum** | ROUGE-L computed over full summaries |\n\n**What to expect:**\n- The base student (1B, no training) provides a lower bound since it has not seen the task data\n- The distilled student (1B, KD) should significantly outperform the base student, demonstrating the knowledge transferred from the 3B teacher\n- If the distilled student scores are not much higher than the baseline, try increasing `distillation_temperature`, adjusting `distillation_ratio`, or training for more epochs\n\n---\n\n## Hyperparameters\n\nFor detailed information on all available hyperparameters, recommended values, and tuning guidance, refer to the [Hyperparameter Reference](../manage-customization-jobs/hyperparameters.md).\n\n---\n\n## Troubleshooting\n\n**Job fails during model download:**\n- Verify authentication secrets are configured (refer to [Managing Secrets](../../get-started/concepts/manage-secrets.md))\n- For gated Hugging Face models (Llama, Gemma), accept the license on the model page\n- Check both `model` (student) and `teacher_model` URNs are correct\n- Ensure both model entities exist: `client.models.retrieve(name=..., workspace=\"default\")`\n\n**Job fails with OOM (Out of Memory) error:**\n\nKD loads both models, so OOM is more likely than with SFT:\n1. **First try:** Use `teacher_precision=\"bf16\"` to reduce teacher memory\n2. **Still OOM:** Reduce `micro_batch_size` to 1\n3. **Still OOM:** Reduce `global_batch_size` and `max_seq_length`\n4. **Last resort:** Increase `num_gpus_per_node`\n\n**No chat template / `/chat/completions` fails:**\n- Use Instruct model variants (e.g., `Llama-3.2-1B-Instruct`) instead of base models (`Llama-3.2-1B`). Base models do not include a chat template in their tokenizer, so the output model will also lack one.\n\n**Distilled model quality is poor:**\n- Increase `distillation_temperature` (try 2.0–5.0) to transfer more nuanced knowledge\n- Adjust `distillation_ratio`—if dataset labels are high-quality, lower the ratio; if the teacher is strong, raise it\n- Increase `epochs` or `max_steps` for more training\n- Verify teacher and student share the same vocabulary\n\n**Vocabulary mismatch error:**\n- Teacher and student must use the same tokenizer. Use models from the same family (e.g., Llama 3.2 1B Instruct + Llama 3.2 3B Instruct)\n\n**Deployment fails:**\n- Verify output model exists: `client.models.retrieve(name=DISTILLED_STUDENT_NAME, workspace=\"default\")`\n- Check job logs: `client.jobs.get_logs(name=kd_job.job.name, workspace=\"default\")`\n- Check student deployment status: `client.inference.deployments.retrieve(name=student_deployment.name, workspace=\"default\")`\n- The distilled model has the same size as the student, so GPU requirements match the student model\n\n\n## Next Steps\n\n- [Monitor training metrics](fine-tune-metrics) in detail\n- [Evaluate your fine-tuned model](../../evaluator/index) using the Evaluator service\n- Learn about [LoRA customization](./lora-customization-job) for resource-efficient fine-tuning\n- Learn about [Full SFT](./sft-customization-job) for direct supervised fine-tuning",
+ "source_html": "Interpreting ROUGE Scores:
\n\n\n\n| Metric | \nMeasures | \n
\n\n\n\n| ROUGE-1 | \nUnigram overlap between prediction and reference | \n
\n\n| ROUGE-2 | \nBigram overlap (captures phrase-level similarity) | \n
\n\n| ROUGE-L | \nLongest common subsequence (captures sentence structure) | \n
\n\n| ROUGE-Lsum | \nROUGE-L computed over full summaries | \n
\n\n
\nWhat to expect:
\n\n- The base student (1B, no training) provides a lower bound since it has not seen the task data
\n- The distilled student (1B, KD) should significantly outperform the base student, demonstrating the knowledge transferred from the 3B teacher
\n- If the distilled student scores are not much higher than the baseline, try increasing
distillation_temperature, adjusting distillation_ratio, or training for more epochs \n
\n
\nHyperparameters
\nFor detailed information on all available hyperparameters, recommended values, and tuning guidance, refer to the Hyperparameter Reference.
\n
\nTroubleshooting
\nJob fails during model download:
\n\n- Verify authentication secrets are configured (refer to Managing Secrets)
\n- For gated Hugging Face models (Llama, Gemma), accept the license on the model page
\n- Check both
model (student) and teacher_model URNs are correct \n- Ensure both model entities exist:
client.models.retrieve(name=..., workspace="default") \n
\nJob fails with OOM (Out of Memory) error:
\nKD loads both models, so OOM is more likely than with SFT:
\n\n- First try: Use
teacher_precision="bf16" to reduce teacher memory \n- Still OOM: Reduce
micro_batch_size to 1 \n- Still OOM: Reduce
global_batch_size and max_seq_length \n- Last resort: Increase
num_gpus_per_node \n
\nNo chat template / /chat/completions fails:
\n\n- Use Instruct model variants (e.g.,
Llama-3.2-1B-Instruct) instead of base models (Llama-3.2-1B). Base models do not include a chat template in their tokenizer, so the output model will also lack one. \n
\nDistilled model quality is poor:
\n\n- Increase
distillation_temperature (try 2.0–5.0) to transfer more nuanced knowledge \n- Adjust
distillation_ratio—if dataset labels are high-quality, lower the ratio; if the teacher is strong, raise it \n- Increase
epochs or max_steps for more training \n- Verify teacher and student share the same vocabulary
\n
\nVocabulary mismatch error:
\n\n- Teacher and student must use the same tokenizer. Use models from the same family (e.g., Llama 3.2 1B Instruct + Llama 3.2 3B Instruct)
\n
\nDeployment fails:
\n\n- Verify output model exists:
client.models.retrieve(name=DISTILLED_STUDENT_NAME, workspace="default") \n- Check job logs:
client.jobs.get_logs(name=kd_job.job.name, workspace="default") \n- Check student deployment status:
client.inference.deployments.retrieve(name=student_deployment.name, workspace="default") \n- The distilled model has the same size as the student, so GPU requirements match the student model
\n
\nNext Steps
\n\n"
}
] };
diff --git a/docs/fern/components/notebooks/dpo-customization-job.json b/docs/fern/components/notebooks/dpo-customization-job.json
index ab69dab155..01a864b23a 100644
--- a/docs/fern/components/notebooks/dpo-customization-job.json
+++ b/docs/fern/components/notebooks/dpo-customization-job.json
@@ -7,8 +7,8 @@
},
{
"type": "markdown",
- "source": "## Prerequisites\n\nBefore starting this tutorial, ensure you have:\n\n1. **Completed the [Quickstart](/documentation/get-started)** to install the NeMo Platform and Python SDK.\n2. **Installed the Python SDK** (PyPI wrapper: `pip install \"nemo-platform[all]\"`; source checkout: run `make bootstrap` from the repository root).\n3. **Installed the `datasets` package**: `pip install datasets`.\n4. **A platform configured with `platform.runtime: kubernetes`.** The `rl` (DPO) backend provisions a Ray cluster and has **no local Docker fallback** — `submit` fails fast on a Docker-runtime platform. Multi-node jobs (`parallelism.num_nodes > 1`) additionally require the platform-side `NMP_RL_MULTINODE_SHARED_STORAGE_PATH`.\n5. **A Hugging Face token** with access to the gated base model (this tutorial uses `meta-llama/Llama-3.2-1B-Instruct`). Export it as `HF_TOKEN`.\n6. **At least one GPU with CUDA 13+** and a GPU execution profile (`nemo jobs list-execution-profiles`).",
- "source_html": "Prerequisites
\nBefore starting this tutorial, ensure you have:
\n\n- Completed the Quickstart to install the NeMo Platform and Python SDK.
\n- Installed the Python SDK (PyPI wrapper:
pip install "nemo-platform[all]"; source checkout: run make bootstrap from the repository root). \n- Installed the
datasets package: pip install datasets. \n- A platform configured with
platform.runtime: kubernetes. The rl (DPO) backend provisions a Ray cluster and has no local Docker fallback — submit fails fast on a Docker-runtime platform. Multi-node jobs (parallelism.num_nodes > 1) additionally require the platform-side NMP_RL_MULTINODE_SHARED_STORAGE_PATH. \n- A Hugging Face token with access to the gated base model (this tutorial uses
meta-llama/Llama-3.2-1B-Instruct). Export it as HF_TOKEN. \n- At least one GPU with CUDA 13+ and a GPU execution profile (
nemo jobs list-execution-profiles). \n
\n"
+ "source": "## Prerequisites\n\nBefore starting this tutorial, ensure you have:\n\n1. **Completed the [Quickstart](/documentation/get-started)** to install the NeMo Platform and Python SDK.\n2. **Installed the Python SDK** (PyPI wrapper: `pip install \"nemo-platform[all,nemo-rl-plugin]\"` so `RlJobInput` is available; source checkout: run `make bootstrap` from the repository root). `nemo-platform[all]` does not include the RL plugin.\n3. **Installed the `datasets` package**: `pip install datasets`.\n4. **A platform configured with `platform.runtime: kubernetes`.** The `rl` (DPO) backend provisions a Ray cluster and has **no local Docker fallback** — `submit` fails fast on a Docker-runtime platform. Multi-node jobs (`parallelism.num_nodes > 1`) additionally require the platform-side `NMP_RL_MULTINODE_SHARED_STORAGE_PATH`.\n5. **A Hugging Face token** with access to the gated base model (this tutorial uses `meta-llama/Llama-3.2-1B-Instruct`). Export it as `HF_TOKEN`.\n6. **At least one GPU with CUDA 13+** and a GPU execution profile (`nemo jobs list-execution-profiles`).",
+ "source_html": "Prerequisites
\nBefore starting this tutorial, ensure you have:
\n\n- Completed the Quickstart to install the NeMo Platform and Python SDK.
\n- Installed the Python SDK (PyPI wrapper:
pip install "nemo-platform[all,nemo-rl-plugin]" so RlJobInput is available; source checkout: run make bootstrap from the repository root). nemo-platform[all] does not include the RL plugin. \n- Installed the
datasets package: pip install datasets. \n- A platform configured with
platform.runtime: kubernetes. The rl (DPO) backend provisions a Ray cluster and has no local Docker fallback — submit fails fast on a Docker-runtime platform. Multi-node jobs (parallelism.num_nodes > 1) additionally require the platform-side NMP_RL_MULTINODE_SHARED_STORAGE_PATH. \n- A Hugging Face token with access to the gated base model (this tutorial uses
meta-llama/Llama-3.2-1B-Instruct). Export it as HF_TOKEN. \n- At least one GPU with CUDA 13+ and a GPU execution profile (
nemo jobs list-execution-profiles). \n
\n"
},
{
"type": "markdown",
diff --git a/docs/fern/components/notebooks/dpo-customization-job.ts b/docs/fern/components/notebooks/dpo-customization-job.ts
index fc8e558903..c0a3dc97ac 100644
--- a/docs/fern/components/notebooks/dpo-customization-job.ts
+++ b/docs/fern/components/notebooks/dpo-customization-job.ts
@@ -12,8 +12,8 @@ export default { cells: [
},
{
"type": "markdown",
- "source": "## Prerequisites\n\nBefore starting this tutorial, ensure you have:\n\n1. **Completed the [Quickstart](/documentation/get-started)** to install the NeMo Platform and Python SDK.\n2. **Installed the Python SDK** (PyPI wrapper: `pip install \"nemo-platform[all]\"`; source checkout: run `make bootstrap` from the repository root).\n3. **Installed the `datasets` package**: `pip install datasets`.\n4. **A platform configured with `platform.runtime: kubernetes`.** The `rl` (DPO) backend provisions a Ray cluster and has **no local Docker fallback** — `submit` fails fast on a Docker-runtime platform. Multi-node jobs (`parallelism.num_nodes > 1`) additionally require the platform-side `NMP_RL_MULTINODE_SHARED_STORAGE_PATH`.\n5. **A Hugging Face token** with access to the gated base model (this tutorial uses `meta-llama/Llama-3.2-1B-Instruct`). Export it as `HF_TOKEN`.\n6. **At least one GPU with CUDA 13+** and a GPU execution profile (`nemo jobs list-execution-profiles`).",
- "source_html": "Prerequisites
\nBefore starting this tutorial, ensure you have:
\n\n- Completed the Quickstart to install the NeMo Platform and Python SDK.
\n- Installed the Python SDK (PyPI wrapper:
pip install "nemo-platform[all]"; source checkout: run make bootstrap from the repository root). \n- Installed the
datasets package: pip install datasets. \n- A platform configured with
platform.runtime: kubernetes. The rl (DPO) backend provisions a Ray cluster and has no local Docker fallback — submit fails fast on a Docker-runtime platform. Multi-node jobs (parallelism.num_nodes > 1) additionally require the platform-side NMP_RL_MULTINODE_SHARED_STORAGE_PATH. \n- A Hugging Face token with access to the gated base model (this tutorial uses
meta-llama/Llama-3.2-1B-Instruct). Export it as HF_TOKEN. \n- At least one GPU with CUDA 13+ and a GPU execution profile (
nemo jobs list-execution-profiles). \n
\n"
+ "source": "## Prerequisites\n\nBefore starting this tutorial, ensure you have:\n\n1. **Completed the [Quickstart](/documentation/get-started)** to install the NeMo Platform and Python SDK.\n2. **Installed the Python SDK** (PyPI wrapper: `pip install \"nemo-platform[all,nemo-rl-plugin]\"` so `RlJobInput` is available; source checkout: run `make bootstrap` from the repository root). `nemo-platform[all]` does not include the RL plugin.\n3. **Installed the `datasets` package**: `pip install datasets`.\n4. **A platform configured with `platform.runtime: kubernetes`.** The `rl` (DPO) backend provisions a Ray cluster and has **no local Docker fallback** — `submit` fails fast on a Docker-runtime platform. Multi-node jobs (`parallelism.num_nodes > 1`) additionally require the platform-side `NMP_RL_MULTINODE_SHARED_STORAGE_PATH`.\n5. **A Hugging Face token** with access to the gated base model (this tutorial uses `meta-llama/Llama-3.2-1B-Instruct`). Export it as `HF_TOKEN`.\n6. **At least one GPU with CUDA 13+** and a GPU execution profile (`nemo jobs list-execution-profiles`).",
+ "source_html": "Prerequisites
\nBefore starting this tutorial, ensure you have:
\n\n- Completed the Quickstart to install the NeMo Platform and Python SDK.
\n- Installed the Python SDK (PyPI wrapper:
pip install "nemo-platform[all,nemo-rl-plugin]" so RlJobInput is available; source checkout: run make bootstrap from the repository root). nemo-platform[all] does not include the RL plugin. \n- Installed the
datasets package: pip install datasets. \n- A platform configured with
platform.runtime: kubernetes. The rl (DPO) backend provisions a Ray cluster and has no local Docker fallback — submit fails fast on a Docker-runtime platform. Multi-node jobs (parallelism.num_nodes > 1) additionally require the platform-side NMP_RL_MULTINODE_SHARED_STORAGE_PATH. \n- A Hugging Face token with access to the gated base model (this tutorial uses
meta-llama/Llama-3.2-1B-Instruct). Export it as HF_TOKEN. \n- At least one GPU with CUDA 13+ and a GPU execution profile (
nemo jobs list-execution-profiles). \n
\n"
},
{
"type": "markdown",
diff --git a/docs/fern/components/notebooks/embedding-customization-job.json b/docs/fern/components/notebooks/embedding-customization-job.json
index 82e62a10c9..05b9ae4c21 100644
--- a/docs/fern/components/notebooks/embedding-customization-job.json
+++ b/docs/fern/components/notebooks/embedding-customization-job.json
@@ -27,10 +27,9 @@
"source_html": "2. Establish Baseline Performance
\nBefore fine-tuning, establish baseline performance with the pretrained model. Deploy it, run a test query, and observe where it struggles. Following fine-tuning, compare the results.
\nScenario: Searching scientific papers by meaning, not keywords.
\nDemo setup:
\n\n- Query: "Conditional Random Fields" (CRFs) - a method for sequence labeling in NLP
\n- Trap: "Random Forests" shares the word "random" but is an unrelated tree-based algorithm
\n- Goal: The model should distinguish between them.
\n
\n"
},
{
- "type": "code",
- "source": "# Install required packages for dataset preparation\n%pip install -q datasets huggingface_hub",
- "language": "python",
- "source_html": "# Install required packages for dataset preparation\n%pip install -q datasets huggingface_hub\n"
+ "type": "markdown",
+ "source": "Install required packages for dataset preparation if they are not installed in your Python environment:\n\n```sh\npip install datasets huggingface_hub\n```",
+ "source_html": "Install required packages for dataset preparation if they are not installed in your Python environment:
\npip install datasets huggingface_hub\n
\n"
},
{
"type": "code",
diff --git a/docs/fern/components/notebooks/embedding-customization-job.ts b/docs/fern/components/notebooks/embedding-customization-job.ts
index 50c25536cf..f6d8265362 100644
--- a/docs/fern/components/notebooks/embedding-customization-job.ts
+++ b/docs/fern/components/notebooks/embedding-customization-job.ts
@@ -32,10 +32,9 @@ export default { cells: [
"source_html": "2. Establish Baseline Performance
\nBefore fine-tuning, establish baseline performance with the pretrained model. Deploy it, run a test query, and observe where it struggles. Following fine-tuning, compare the results.
\nScenario: Searching scientific papers by meaning, not keywords.
\nDemo setup:
\n\n- Query: "Conditional Random Fields" (CRFs) - a method for sequence labeling in NLP
\n- Trap: "Random Forests" shares the word "random" but is an unrelated tree-based algorithm
\n- Goal: The model should distinguish between them.
\n
\n"
},
{
- "type": "code",
- "source": "# Install required packages for dataset preparation\n%pip install -q datasets huggingface_hub",
- "language": "python",
- "source_html": "# Install required packages for dataset preparation\n%pip install -q datasets huggingface_hub\n"
+ "type": "markdown",
+ "source": "Install required packages for dataset preparation if they are not installed in your Python environment:\n\n```sh\npip install datasets huggingface_hub\n```",
+ "source_html": "Install required packages for dataset preparation if they are not installed in your Python environment:
\npip install datasets huggingface_hub\n
\n"
},
{
"type": "code",
diff --git a/docs/fern/components/notebooks/sft-customization-job.json b/docs/fern/components/notebooks/sft-customization-job.json
index 33c6a61148..471dcedea5 100644
--- a/docs/fern/components/notebooks/sft-customization-job.json
+++ b/docs/fern/components/notebooks/sft-customization-job.json
@@ -183,8 +183,8 @@
},
{
"type": "markdown",
- "source": "#### Evaluation Best Practices\n\n**Manual Evaluation** (Recommended)\n- Test with real-world examples from your use case\n- Compare responses to base model and expected outputs\n- Verify the model exhibits desired behavior changes\n- Check edge cases and error handling\n\n**What to look for:**\n- ✅ Model follows your desired output format\n- ✅ Applies domain knowledge correctly\n- ✅ Maintains general language capabilities\n- ✅ Avoids unwanted behaviors or biases\n- ❌ Doesn't hallucinate facts not in training data\n- ❌ Doesn't produce repetitive or nonsensical outputs\n\n---\n\n## Hyperparameters\n\nFor detailed information on all available hyperparameters, recommended values, and tuning guidance, refer to the [Hyperparameter Reference](../manage-customization-jobs/hyperparameters.md).\n\n---\n\n\n## Troubleshooting\n\n**Job fails during model download:**\n- Verify authentication secrets are configured (refer to [Managing Secrets](../../get-started/concepts/manage-secrets.md))\n- For gated Hugging Face models (Llama, Gemma), accept the license on the model page (for example, [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct))\n- Confirm the model fileset uses `token_secret=hf_secret.name` for gated models\n- Check `AutomodelJobInput` references use the `workspace/name` format: `model=f\"default/{MODEL_NAME}\"` and `dataset={\"training\": f\"default/{DATASET_NAME}\"}` (for example, `default/llama-3-2-1b-base`, `default/sft-dataset`)\n- Verify the model entity points at the fileset: `fileset=f\"default/{MODEL_NAME}\"`\n- Check job status: `client.jobs.get_status(name=job.job.name, workspace=\"default\")`\n\n**Job fails with OOM (Out of Memory) error:**\n1. **First try:** Reduce `global_batch_size` from 64 to 32 or 16 in `batch={...}`\n2. **Still OOM:** Keep `micro_batch_size` at 1 (already the minimum in this tutorial)\n3. **Still OOM:** Reduce `max_seq_length` from 2048 to 1024 or 512 in `training={...}`\n4. **Last resort:** Increase `num_gpus_per_node` and `tensor_parallel_size` in `parallelism={...}`\n\n**Loss curves not decreasing (underfitting):**\n- Increase training duration: raise `epochs` from 2 to 3-5 in `schedule={...}`\n- Adjust learning rate: try `1e-4` or `1e-5` instead of the default `5e-5` in `optimizer={...}`\n- Check data quality: Verify formatting, remove duplicates, ensure diversity\n\n**Training loss decreases but validation loss increases (overfitting):**\n- Reduce `epochs` from 2 to 1 in `schedule={...}`\n- Lower `learning_rate` from `5e-5` to `2e-5` or `1e-5` in `optimizer={...}`\n- Increase dataset size and diversity\n- Verify train/validation split has no data leakage\n\n**Model output quality is poor despite good training metrics:**\n- Training metrics optimize for loss, not your actual task—evaluate on real use cases\n- Review data quality, format, and diversity—metrics can be misleading with poor data\n- Try a different base model size or architecture\n- Adjust `learning_rate` and `global_batch_size`\n- Compare to baseline: Test base model to ensure fine-tuning improved performance\n\n**Deployment fails:**\n- Verify output model exists: `client.models.retrieve(name=OUTPUT_NAME, workspace=\"default\")`\n- Check deployment logs: `client.inference.deployments.get_logs(name=deployment.name, workspace=\"default\")`\n- Ensure sufficient GPU resources for `executor_config={\"gpu\": 1, ...}`\n- Verify the deployment config matches this tutorial: `engine=\"vllm\"` with `vllm/vllm-openai:v0.22.1`\n\n\n## Next Steps\n\n- [Monitor training metrics](fine-tune-metrics) in detail\n- [Evaluate your fine-tuned model](../../evaluator/index) using the Evaluator service\n- Learn about [LoRA customization](./lora-customization-job) for resource-efficient fine-tuning",
- "source_html": "Evaluation Best Practices
\nManual Evaluation (Recommended)
\n\n- Test with real-world examples from your use case
\n- Compare responses to base model and expected outputs
\n- Verify the model exhibits desired behavior changes
\n- Check edge cases and error handling
\n
\nWhat to look for:
\n\n- ✅ Model follows your desired output format
\n- ✅ Applies domain knowledge correctly
\n- ✅ Maintains general language capabilities
\n- ✅ Avoids unwanted behaviors or biases
\n- ❌ Doesn't hallucinate facts not in training data
\n- ❌ Doesn't produce repetitive or nonsensical outputs
\n
\n
\nHyperparameters
\nFor detailed information on all available hyperparameters, recommended values, and tuning guidance, refer to the Hyperparameter Reference.
\n
\nTroubleshooting
\nJob fails during model download:
\n\n- Verify authentication secrets are configured (refer to Managing Secrets)
\n- For gated Hugging Face models (Llama, Gemma), accept the license on the model page (for example, meta-llama/Llama-3.2-1B-Instruct)
\n- Confirm the model fileset uses
token_secret=hf_secret.name for gated models \n- Check
AutomodelJobInput references use the workspace/name format: model=f"default/{MODEL_NAME}" and dataset={"training": f"default/{DATASET_NAME}"} (for example, default/llama-3-2-1b-base, default/sft-dataset) \n- Verify the model entity points at the fileset:
fileset=f"default/{MODEL_NAME}" \n- Check job status:
client.jobs.get_status(name=job.job.name, workspace="default") \n
\nJob fails with OOM (Out of Memory) error:
\n\n- First try: Reduce
global_batch_size from 64 to 32 or 16 in batch={...} \n- Still OOM: Keep
micro_batch_size at 1 (already the minimum in this tutorial) \n- Still OOM: Reduce
max_seq_length from 2048 to 1024 or 512 in training={...} \n- Last resort: Increase
num_gpus_per_node and tensor_parallel_size in parallelism={...} \n
\nLoss curves not decreasing (underfitting):
\n\n- Increase training duration: raise
epochs from 2 to 3-5 in schedule={...} \n- Adjust learning rate: try
1e-4 or 1e-5 instead of the default 5e-5 in optimizer={...} \n- Check data quality: Verify formatting, remove duplicates, ensure diversity
\n
\nTraining loss decreases but validation loss increases (overfitting):
\n\n- Reduce
epochs from 2 to 1 in schedule={...} \n- Lower
learning_rate from 5e-5 to 2e-5 or 1e-5 in optimizer={...} \n- Increase dataset size and diversity
\n- Verify train/validation split has no data leakage
\n
\nModel output quality is poor despite good training metrics:
\n\n- Training metrics optimize for loss, not your actual task—evaluate on real use cases
\n- Review data quality, format, and diversity—metrics can be misleading with poor data
\n- Try a different base model size or architecture
\n- Adjust
learning_rate and global_batch_size \n- Compare to baseline: Test base model to ensure fine-tuning improved performance
\n
\nDeployment fails:
\n\n- Verify output model exists:
client.models.retrieve(name=OUTPUT_NAME, workspace="default") \n- Check deployment logs:
client.inference.deployments.get_logs(name=deployment.name, workspace="default") \n- Ensure sufficient GPU resources for
executor_config={"gpu": 1, ...} \n- Verify the deployment config matches this tutorial:
engine="vllm" with vllm/vllm-openai:v0.22.1 \n
\nNext Steps
\n\n"
+ "source": "#### Evaluation Best Practices\n\n**Manual Evaluation** (Recommended)\n- Test with real-world examples from your use case\n- Compare responses to base model and expected outputs\n- Verify the model exhibits desired behavior changes\n- Check edge cases and error handling\n\n**What to look for:**\n- ✅ Model follows your desired output format\n- ✅ Applies domain knowledge correctly\n- ✅ Maintains general language capabilities\n- ✅ Avoids unwanted behaviors or biases\n- ❌ Doesn't hallucinate facts not in training data\n- ❌ Doesn't produce repetitive or nonsensical outputs\n\n---\n\n## Hyperparameters\n\nFor detailed information on all available hyperparameters, recommended values, and tuning guidance, refer to the [Hyperparameter Reference](../manage-customization-jobs/hyperparameters.md).\n\n---\n\n\n## Troubleshooting\n\n**Job fails during model download:**\n- Verify authentication secrets are configured (refer to [Managing Secrets](../../get-started/concepts/manage-secrets.md))\n- For gated Hugging Face models (Llama, Gemma), accept the license on the model page (for example, [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct))\n- Confirm the model fileset uses `token_secret=hf_secret.name` for gated models\n- Check `AutomodelJobInput` references use the `workspace/name` format: `model=f\"default/{MODEL_NAME}\"` and `dataset={\"training\": f\"default/{DATASET_NAME}\"}` (for example, `default/llama-3-2-1b-base`, `default/sft-dataset`)\n- Verify the model entity points at the fileset: `fileset=f\"default/{MODEL_NAME}\"`\n- Check job status: `client.jobs.get_status(name=job.job.name, workspace=\"default\")`\n\n**Job fails with OOM (Out of Memory) error:**\n1. **First try:** Reduce `global_batch_size` from 64 to 32 or 16 in `batch={...}`\n2. **Still OOM:** Keep `micro_batch_size` at 1 (already the minimum in this tutorial)\n3. **Still OOM:** Reduce `max_seq_length` from 2048 to 1024 or 512 in `training={...}`\n4. **Last resort:** Increase `num_gpus_per_node` and `tensor_parallel_size` in `parallelism={...}`\n\n**Loss curves not decreasing (underfitting):**\n- Increase training duration: raise `epochs` from 2 to 3-5 in `schedule={...}`\n- Adjust learning rate: try `1e-4` or `1e-5` instead of the default `5e-5` in `optimizer={...}`\n- Check data quality: Verify formatting, remove duplicates, ensure diversity\n\n**Training loss decreases but validation loss increases (overfitting):**\n- Reduce `epochs` from 2 to 1 in `schedule={...}`\n- Lower `learning_rate` from `5e-5` to `2e-5` or `1e-5` in `optimizer={...}`\n- Increase dataset size and diversity\n- Verify train/validation split has no data leakage\n\n**Model output quality is poor despite good training metrics:**\n- Training metrics optimize for loss, not your actual task—evaluate on real use cases\n- Review data quality, format, and diversity—metrics can be misleading with poor data\n- Try a different base model size or architecture\n- Adjust `learning_rate` and `global_batch_size`\n- Compare to baseline: Test base model to ensure fine-tuning improved performance\n\n**Deployment fails:**\n- Verify output model exists: `client.models.retrieve(name=OUTPUT_NAME, workspace=\"default\")`\n- Check job logs: `client.jobs.get_logs(name=job.job.name, workspace=\"default\")`\n- Check deployment status: `client.inference.deployments.retrieve(name=deployment.name, workspace=\"default\")`\n- Ensure sufficient GPU resources for `executor_config={\"gpu\": 1, ...}`\n- Verify the deployment config matches this tutorial: `engine=\"vllm\"` with `vllm/vllm-openai:v0.22.1`\n\n\n## Next Steps\n\n- [Monitor training metrics](fine-tune-metrics) in detail\n- [Evaluate your fine-tuned model](../../evaluator/index) using the Evaluator service\n- Learn about [LoRA customization](./lora-customization-job) for resource-efficient fine-tuning",
+ "source_html": "Evaluation Best Practices
\nManual Evaluation (Recommended)
\n\n- Test with real-world examples from your use case
\n- Compare responses to base model and expected outputs
\n- Verify the model exhibits desired behavior changes
\n- Check edge cases and error handling
\n
\nWhat to look for:
\n\n- ✅ Model follows your desired output format
\n- ✅ Applies domain knowledge correctly
\n- ✅ Maintains general language capabilities
\n- ✅ Avoids unwanted behaviors or biases
\n- ❌ Doesn't hallucinate facts not in training data
\n- ❌ Doesn't produce repetitive or nonsensical outputs
\n
\n
\nHyperparameters
\nFor detailed information on all available hyperparameters, recommended values, and tuning guidance, refer to the Hyperparameter Reference.
\n
\nTroubleshooting
\nJob fails during model download:
\n\n- Verify authentication secrets are configured (refer to Managing Secrets)
\n- For gated Hugging Face models (Llama, Gemma), accept the license on the model page (for example, meta-llama/Llama-3.2-1B-Instruct)
\n- Confirm the model fileset uses
token_secret=hf_secret.name for gated models \n- Check
AutomodelJobInput references use the workspace/name format: model=f"default/{MODEL_NAME}" and dataset={"training": f"default/{DATASET_NAME}"} (for example, default/llama-3-2-1b-base, default/sft-dataset) \n- Verify the model entity points at the fileset:
fileset=f"default/{MODEL_NAME}" \n- Check job status:
client.jobs.get_status(name=job.job.name, workspace="default") \n
\nJob fails with OOM (Out of Memory) error:
\n\n- First try: Reduce
global_batch_size from 64 to 32 or 16 in batch={...} \n- Still OOM: Keep
micro_batch_size at 1 (already the minimum in this tutorial) \n- Still OOM: Reduce
max_seq_length from 2048 to 1024 or 512 in training={...} \n- Last resort: Increase
num_gpus_per_node and tensor_parallel_size in parallelism={...} \n
\nLoss curves not decreasing (underfitting):
\n\n- Increase training duration: raise
epochs from 2 to 3-5 in schedule={...} \n- Adjust learning rate: try
1e-4 or 1e-5 instead of the default 5e-5 in optimizer={...} \n- Check data quality: Verify formatting, remove duplicates, ensure diversity
\n
\nTraining loss decreases but validation loss increases (overfitting):
\n\n- Reduce
epochs from 2 to 1 in schedule={...} \n- Lower
learning_rate from 5e-5 to 2e-5 or 1e-5 in optimizer={...} \n- Increase dataset size and diversity
\n- Verify train/validation split has no data leakage
\n
\nModel output quality is poor despite good training metrics:
\n\n- Training metrics optimize for loss, not your actual task—evaluate on real use cases
\n- Review data quality, format, and diversity—metrics can be misleading with poor data
\n- Try a different base model size or architecture
\n- Adjust
learning_rate and global_batch_size \n- Compare to baseline: Test base model to ensure fine-tuning improved performance
\n
\nDeployment fails:
\n\n- Verify output model exists:
client.models.retrieve(name=OUTPUT_NAME, workspace="default") \n- Check job logs:
client.jobs.get_logs(name=job.job.name, workspace="default") \n- Check deployment status:
client.inference.deployments.retrieve(name=deployment.name, workspace="default") \n- Ensure sufficient GPU resources for
executor_config={"gpu": 1, ...} \n- Verify the deployment config matches this tutorial:
engine="vllm" with vllm/vllm-openai:v0.22.1 \n
\nNext Steps
\n\n"
}
]
}
\ No newline at end of file
diff --git a/docs/fern/components/notebooks/sft-customization-job.ts b/docs/fern/components/notebooks/sft-customization-job.ts
index 5b804ed7f2..a57f08e764 100644
--- a/docs/fern/components/notebooks/sft-customization-job.ts
+++ b/docs/fern/components/notebooks/sft-customization-job.ts
@@ -188,7 +188,7 @@ export default { cells: [
},
{
"type": "markdown",
- "source": "#### Evaluation Best Practices\n\n**Manual Evaluation** (Recommended)\n- Test with real-world examples from your use case\n- Compare responses to base model and expected outputs\n- Verify the model exhibits desired behavior changes\n- Check edge cases and error handling\n\n**What to look for:**\n- ✅ Model follows your desired output format\n- ✅ Applies domain knowledge correctly\n- ✅ Maintains general language capabilities\n- ✅ Avoids unwanted behaviors or biases\n- ❌ Doesn't hallucinate facts not in training data\n- ❌ Doesn't produce repetitive or nonsensical outputs\n\n---\n\n## Hyperparameters\n\nFor detailed information on all available hyperparameters, recommended values, and tuning guidance, refer to the [Hyperparameter Reference](../manage-customization-jobs/hyperparameters.md).\n\n---\n\n\n## Troubleshooting\n\n**Job fails during model download:**\n- Verify authentication secrets are configured (refer to [Managing Secrets](../../get-started/concepts/manage-secrets.md))\n- For gated Hugging Face models (Llama, Gemma), accept the license on the model page (for example, [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct))\n- Confirm the model fileset uses `token_secret=hf_secret.name` for gated models\n- Check `AutomodelJobInput` references use the `workspace/name` format: `model=f\"default/{MODEL_NAME}\"` and `dataset={\"training\": f\"default/{DATASET_NAME}\"}` (for example, `default/llama-3-2-1b-base`, `default/sft-dataset`)\n- Verify the model entity points at the fileset: `fileset=f\"default/{MODEL_NAME}\"`\n- Check job status: `client.jobs.get_status(name=job.job.name, workspace=\"default\")`\n\n**Job fails with OOM (Out of Memory) error:**\n1. **First try:** Reduce `global_batch_size` from 64 to 32 or 16 in `batch={...}`\n2. **Still OOM:** Keep `micro_batch_size` at 1 (already the minimum in this tutorial)\n3. **Still OOM:** Reduce `max_seq_length` from 2048 to 1024 or 512 in `training={...}`\n4. **Last resort:** Increase `num_gpus_per_node` and `tensor_parallel_size` in `parallelism={...}`\n\n**Loss curves not decreasing (underfitting):**\n- Increase training duration: raise `epochs` from 2 to 3-5 in `schedule={...}`\n- Adjust learning rate: try `1e-4` or `1e-5` instead of the default `5e-5` in `optimizer={...}`\n- Check data quality: Verify formatting, remove duplicates, ensure diversity\n\n**Training loss decreases but validation loss increases (overfitting):**\n- Reduce `epochs` from 2 to 1 in `schedule={...}`\n- Lower `learning_rate` from `5e-5` to `2e-5` or `1e-5` in `optimizer={...}`\n- Increase dataset size and diversity\n- Verify train/validation split has no data leakage\n\n**Model output quality is poor despite good training metrics:**\n- Training metrics optimize for loss, not your actual task—evaluate on real use cases\n- Review data quality, format, and diversity—metrics can be misleading with poor data\n- Try a different base model size or architecture\n- Adjust `learning_rate` and `global_batch_size`\n- Compare to baseline: Test base model to ensure fine-tuning improved performance\n\n**Deployment fails:**\n- Verify output model exists: `client.models.retrieve(name=OUTPUT_NAME, workspace=\"default\")`\n- Check deployment logs: `client.inference.deployments.get_logs(name=deployment.name, workspace=\"default\")`\n- Ensure sufficient GPU resources for `executor_config={\"gpu\": 1, ...}`\n- Verify the deployment config matches this tutorial: `engine=\"vllm\"` with `vllm/vllm-openai:v0.22.1`\n\n\n## Next Steps\n\n- [Monitor training metrics](fine-tune-metrics) in detail\n- [Evaluate your fine-tuned model](../../evaluator/index) using the Evaluator service\n- Learn about [LoRA customization](./lora-customization-job) for resource-efficient fine-tuning",
- "source_html": "Evaluation Best Practices
\nManual Evaluation (Recommended)
\n\n- Test with real-world examples from your use case
\n- Compare responses to base model and expected outputs
\n- Verify the model exhibits desired behavior changes
\n- Check edge cases and error handling
\n
\nWhat to look for:
\n\n- ✅ Model follows your desired output format
\n- ✅ Applies domain knowledge correctly
\n- ✅ Maintains general language capabilities
\n- ✅ Avoids unwanted behaviors or biases
\n- ❌ Doesn't hallucinate facts not in training data
\n- ❌ Doesn't produce repetitive or nonsensical outputs
\n
\n
\nHyperparameters
\nFor detailed information on all available hyperparameters, recommended values, and tuning guidance, refer to the Hyperparameter Reference.
\n
\nTroubleshooting
\nJob fails during model download:
\n\n- Verify authentication secrets are configured (refer to Managing Secrets)
\n- For gated Hugging Face models (Llama, Gemma), accept the license on the model page (for example, meta-llama/Llama-3.2-1B-Instruct)
\n- Confirm the model fileset uses
token_secret=hf_secret.name for gated models \n- Check
AutomodelJobInput references use the workspace/name format: model=f"default/{MODEL_NAME}" and dataset={"training": f"default/{DATASET_NAME}"} (for example, default/llama-3-2-1b-base, default/sft-dataset) \n- Verify the model entity points at the fileset:
fileset=f"default/{MODEL_NAME}" \n- Check job status:
client.jobs.get_status(name=job.job.name, workspace="default") \n
\nJob fails with OOM (Out of Memory) error:
\n\n- First try: Reduce
global_batch_size from 64 to 32 or 16 in batch={...} \n- Still OOM: Keep
micro_batch_size at 1 (already the minimum in this tutorial) \n- Still OOM: Reduce
max_seq_length from 2048 to 1024 or 512 in training={...} \n- Last resort: Increase
num_gpus_per_node and tensor_parallel_size in parallelism={...} \n
\nLoss curves not decreasing (underfitting):
\n\n- Increase training duration: raise
epochs from 2 to 3-5 in schedule={...} \n- Adjust learning rate: try
1e-4 or 1e-5 instead of the default 5e-5 in optimizer={...} \n- Check data quality: Verify formatting, remove duplicates, ensure diversity
\n
\nTraining loss decreases but validation loss increases (overfitting):
\n\n- Reduce
epochs from 2 to 1 in schedule={...} \n- Lower
learning_rate from 5e-5 to 2e-5 or 1e-5 in optimizer={...} \n- Increase dataset size and diversity
\n- Verify train/validation split has no data leakage
\n
\nModel output quality is poor despite good training metrics:
\n\n- Training metrics optimize for loss, not your actual task—evaluate on real use cases
\n- Review data quality, format, and diversity—metrics can be misleading with poor data
\n- Try a different base model size or architecture
\n- Adjust
learning_rate and global_batch_size \n- Compare to baseline: Test base model to ensure fine-tuning improved performance
\n
\nDeployment fails:
\n\n- Verify output model exists:
client.models.retrieve(name=OUTPUT_NAME, workspace="default") \n- Check deployment logs:
client.inference.deployments.get_logs(name=deployment.name, workspace="default") \n- Ensure sufficient GPU resources for
executor_config={"gpu": 1, ...} \n- Verify the deployment config matches this tutorial:
engine="vllm" with vllm/vllm-openai:v0.22.1 \n
\nNext Steps
\n\n"
+ "source": "#### Evaluation Best Practices\n\n**Manual Evaluation** (Recommended)\n- Test with real-world examples from your use case\n- Compare responses to base model and expected outputs\n- Verify the model exhibits desired behavior changes\n- Check edge cases and error handling\n\n**What to look for:**\n- ✅ Model follows your desired output format\n- ✅ Applies domain knowledge correctly\n- ✅ Maintains general language capabilities\n- ✅ Avoids unwanted behaviors or biases\n- ❌ Doesn't hallucinate facts not in training data\n- ❌ Doesn't produce repetitive or nonsensical outputs\n\n---\n\n## Hyperparameters\n\nFor detailed information on all available hyperparameters, recommended values, and tuning guidance, refer to the [Hyperparameter Reference](../manage-customization-jobs/hyperparameters.md).\n\n---\n\n\n## Troubleshooting\n\n**Job fails during model download:**\n- Verify authentication secrets are configured (refer to [Managing Secrets](../../get-started/concepts/manage-secrets.md))\n- For gated Hugging Face models (Llama, Gemma), accept the license on the model page (for example, [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct))\n- Confirm the model fileset uses `token_secret=hf_secret.name` for gated models\n- Check `AutomodelJobInput` references use the `workspace/name` format: `model=f\"default/{MODEL_NAME}\"` and `dataset={\"training\": f\"default/{DATASET_NAME}\"}` (for example, `default/llama-3-2-1b-base`, `default/sft-dataset`)\n- Verify the model entity points at the fileset: `fileset=f\"default/{MODEL_NAME}\"`\n- Check job status: `client.jobs.get_status(name=job.job.name, workspace=\"default\")`\n\n**Job fails with OOM (Out of Memory) error:**\n1. **First try:** Reduce `global_batch_size` from 64 to 32 or 16 in `batch={...}`\n2. **Still OOM:** Keep `micro_batch_size` at 1 (already the minimum in this tutorial)\n3. **Still OOM:** Reduce `max_seq_length` from 2048 to 1024 or 512 in `training={...}`\n4. **Last resort:** Increase `num_gpus_per_node` and `tensor_parallel_size` in `parallelism={...}`\n\n**Loss curves not decreasing (underfitting):**\n- Increase training duration: raise `epochs` from 2 to 3-5 in `schedule={...}`\n- Adjust learning rate: try `1e-4` or `1e-5` instead of the default `5e-5` in `optimizer={...}`\n- Check data quality: Verify formatting, remove duplicates, ensure diversity\n\n**Training loss decreases but validation loss increases (overfitting):**\n- Reduce `epochs` from 2 to 1 in `schedule={...}`\n- Lower `learning_rate` from `5e-5` to `2e-5` or `1e-5` in `optimizer={...}`\n- Increase dataset size and diversity\n- Verify train/validation split has no data leakage\n\n**Model output quality is poor despite good training metrics:**\n- Training metrics optimize for loss, not your actual task—evaluate on real use cases\n- Review data quality, format, and diversity—metrics can be misleading with poor data\n- Try a different base model size or architecture\n- Adjust `learning_rate` and `global_batch_size`\n- Compare to baseline: Test base model to ensure fine-tuning improved performance\n\n**Deployment fails:**\n- Verify output model exists: `client.models.retrieve(name=OUTPUT_NAME, workspace=\"default\")`\n- Check job logs: `client.jobs.get_logs(name=job.job.name, workspace=\"default\")`\n- Check deployment status: `client.inference.deployments.retrieve(name=deployment.name, workspace=\"default\")`\n- Ensure sufficient GPU resources for `executor_config={\"gpu\": 1, ...}`\n- Verify the deployment config matches this tutorial: `engine=\"vllm\"` with `vllm/vllm-openai:v0.22.1`\n\n\n## Next Steps\n\n- [Monitor training metrics](fine-tune-metrics) in detail\n- [Evaluate your fine-tuned model](../../evaluator/index) using the Evaluator service\n- Learn about [LoRA customization](./lora-customization-job) for resource-efficient fine-tuning",
+ "source_html": "Evaluation Best Practices
\nManual Evaluation (Recommended)
\n\n- Test with real-world examples from your use case
\n- Compare responses to base model and expected outputs
\n- Verify the model exhibits desired behavior changes
\n- Check edge cases and error handling
\n
\nWhat to look for:
\n\n- ✅ Model follows your desired output format
\n- ✅ Applies domain knowledge correctly
\n- ✅ Maintains general language capabilities
\n- ✅ Avoids unwanted behaviors or biases
\n- ❌ Doesn't hallucinate facts not in training data
\n- ❌ Doesn't produce repetitive or nonsensical outputs
\n
\n
\nHyperparameters
\nFor detailed information on all available hyperparameters, recommended values, and tuning guidance, refer to the Hyperparameter Reference.
\n
\nTroubleshooting
\nJob fails during model download:
\n\n- Verify authentication secrets are configured (refer to Managing Secrets)
\n- For gated Hugging Face models (Llama, Gemma), accept the license on the model page (for example, meta-llama/Llama-3.2-1B-Instruct)
\n- Confirm the model fileset uses
token_secret=hf_secret.name for gated models \n- Check
AutomodelJobInput references use the workspace/name format: model=f"default/{MODEL_NAME}" and dataset={"training": f"default/{DATASET_NAME}"} (for example, default/llama-3-2-1b-base, default/sft-dataset) \n- Verify the model entity points at the fileset:
fileset=f"default/{MODEL_NAME}" \n- Check job status:
client.jobs.get_status(name=job.job.name, workspace="default") \n
\nJob fails with OOM (Out of Memory) error:
\n\n- First try: Reduce
global_batch_size from 64 to 32 or 16 in batch={...} \n- Still OOM: Keep
micro_batch_size at 1 (already the minimum in this tutorial) \n- Still OOM: Reduce
max_seq_length from 2048 to 1024 or 512 in training={...} \n- Last resort: Increase
num_gpus_per_node and tensor_parallel_size in parallelism={...} \n
\nLoss curves not decreasing (underfitting):
\n\n- Increase training duration: raise
epochs from 2 to 3-5 in schedule={...} \n- Adjust learning rate: try
1e-4 or 1e-5 instead of the default 5e-5 in optimizer={...} \n- Check data quality: Verify formatting, remove duplicates, ensure diversity
\n
\nTraining loss decreases but validation loss increases (overfitting):
\n\n- Reduce
epochs from 2 to 1 in schedule={...} \n- Lower
learning_rate from 5e-5 to 2e-5 or 1e-5 in optimizer={...} \n- Increase dataset size and diversity
\n- Verify train/validation split has no data leakage
\n
\nModel output quality is poor despite good training metrics:
\n\n- Training metrics optimize for loss, not your actual task—evaluate on real use cases
\n- Review data quality, format, and diversity—metrics can be misleading with poor data
\n- Try a different base model size or architecture
\n- Adjust
learning_rate and global_batch_size \n- Compare to baseline: Test base model to ensure fine-tuning improved performance
\n
\nDeployment fails:
\n\n- Verify output model exists:
client.models.retrieve(name=OUTPUT_NAME, workspace="default") \n- Check job logs:
client.jobs.get_logs(name=job.job.name, workspace="default") \n- Check deployment status:
client.inference.deployments.retrieve(name=deployment.name, workspace="default") \n- Ensure sufficient GPU resources for
executor_config={"gpu": 1, ...} \n- Verify the deployment config matches this tutorial:
engine="vllm" with vllm/vllm-openai:v0.22.1 \n
\nNext Steps
\n\n"
}
] };
diff --git a/docs/fern/versions/latest.yml b/docs/fern/versions/latest.yml
index 7a3ce91903..d7c870a4cb 100644
--- a/docs/fern/versions/latest.yml
+++ b/docs/fern/versions/latest.yml
@@ -213,6 +213,9 @@ navigation:
- page: SFT Customization Job
path: ../../customizer/tutorials/sft-customization-job.mdx
slug: sft-customization-job
+ - page: DPO Customization Job
+ path: ../../customizer/tutorials/dpo-customization-job.mdx
+ slug: dpo-customization-job
- page: LoRA Customization Job
path: ../../customizer/tutorials/lora-customization-job.mdx
slug: lora-customization-job
diff --git a/docs/get-started/concepts/filtering.mdx b/docs/get-started/concepts/filtering.mdx
index e41524373c..8e99441450 100644
--- a/docs/get-started/concepts/filtering.mdx
+++ b/docs/get-started/concepts/filtering.mdx
@@ -185,6 +185,6 @@ metrics = client.evaluation.metrics.list(
```python
jobs = client.jobs.list(
workspace="default",
- filter={"source": "automodel", "status": "completed"},
+ filter={"source": "customization", "status": "completed"},
)
```
diff --git a/docs/troubleshooting/customizer.mdx b/docs/troubleshooting/customizer.mdx
index d1dabbfedb..c59e681959 100644
--- a/docs/troubleshooting/customizer.mdx
+++ b/docs/troubleshooting/customizer.mdx
@@ -49,5 +49,6 @@ Batch and sequence-length fields differ by backend. Use the fully qualified path
**Deployment fails:**
- Verify the base model and adapter exist: `client.models.retrieve(name=MODEL_NAME, workspace="default")` -- the LoRA adapter appears in the base model's `adapters` list, not as a separate model entity
-- Check deployment logs: `client.inference.deployments.get_logs(name=deployment.name, workspace="default")`
+- Check job logs: `client.jobs.get_logs(name=job.name, workspace="default")`
+- Check deployment status: `client.inference.deployments.retrieve(name=DEPLOYMENT_NAME, workspace="default")`
- Ensure sufficient GPU resources for the model size