Fine-Tuning LLMs: Complete Step-by-Step Guide from Data to Deployment

{"messages": [{"role": "system", "content": "You are a helpful customer support assistant."}, {"role": "user", "content": "How do I reset my password?"}, {"role": "assistant", "content": "To reset your password: 1) Click 'Forgot Password' on the login page, 2) Enter your email, 3) Check your email for a reset link, 4) Follow the link to create a new password."}]}
{"messages": [{"role": "system", "content": "You are a helpful customer support assistant."}, {"role": "user", "content": "Can I export my data?"}, {"role": "assistant", "content": "Yes! You can export your data anytime: 1) Go to Settings > Data Export, 2) Choose your format (CSV or JSON), 3) Click 'Export'. Your file will download immediately."}]}

training_examples = []

def create_example(system_msg, user_msg, assistant_msg):
    """Helper to create training examples."""
    return {
        "messages": [
            {"role": "system", "content": system_msg},
            {"role": "user", "content": user_msg},
            {"role": "assistant", "content": assistant_msg}
        ]
    }

# Customer support examples
training_examples.append(create_example(
    system_msg="You are a helpful customer support assistant for TechCorp. Be concise and professional.",
    user_msg="I was charged twice this month!",
    assistant_msg="I apologize for the billing error. I've checked your account and confirmed you were charged twice on Jan 15. I'm processing a refund for the duplicate charge of $49.99, which will appear in 3-5 business days. Is there anything else I can help with?"
))

# Save to JSONL
import json
with open('training_data.jsonl', 'w') as f:
    for example in training_examples:
        f.write(json.dumps(example) + '\n')

import json
import pandas as pd

def convert_support_tickets_to_training_data(tickets_csv):
    """Convert support ticket history to training format."""
    df = pd.read_csv(tickets_csv)
    training_data = []

    for _, row in df.iterrows():
        # Only use tickets with good resolutions
        if row['satisfaction_score'] >= 4:  # 4 or 5 stars
            example = {
                "messages": [
                    {"role": "system", "content": "You are a helpful customer support assistant."},
                    {"role": "user", "content": row['customer_message']},
                    {"role": "assistant", "content": row['agent_response']}
                ]
            }
            training_data.append(example)

    # Write to JSONL
    with open('training_data.jsonl', 'w') as f:
        for example in training_data:
            f.write(json.dumps(example) + '\n')

    return len(training_data)

# Convert your data
num_examples = convert_support_tickets_to_training_data('support_tickets.csv')
print(f"Created {num_examples} training examples")

from openai import OpenAI
import json

client = OpenAI()

def generate_synthetic_examples(scenarios, num_per_scenario=10):
    """Generate synthetic training data using GPT-4."""
    training_data = []

    for scenario in scenarios:
        for _ in range(num_per_scenario):
            # Generate a realistic user query
            query_prompt = f"Generate a realistic customer service question about: {scenario}"
            query_response = client.chat.completions.create(
                model="gpt-4o",
                messages=[{"role": "user", "content": query_prompt}],
                temperature=0.8
            )
            user_query = query_response.choices[0].message.content

            # Generate ideal response
            response_prompt = f"As a customer support expert, provide a helpful, professional response to: {user_query}"
            response = client.chat.completions.create(
                model="gpt-4o",
                messages=[{"role": "user", "content": response_prompt}],
                temperature=0.3
            )
            assistant_response = response.choices[0].message.content

            training_data.append({
                "messages": [
                    {"role": "system", "content": "You are a helpful customer support assistant."},
                    {"role": "user", "content": user_query},
                    {"role": "assistant", "content": assistant_response}
                ]
            })

    return training_data

# Generate data
scenarios = [
    "password reset",
    "billing inquiry",
    "feature request",
    "technical issue",
    "account cancellation"
]

synthetic_data = generate_synthetic_examples(scenarios, num_per_scenario=50)

# Save to file
with open('synthetic_training.jsonl', 'w') as f:
    for example in synthetic_data:
        f.write(json.dumps(example) + '\n')

import json

def validate_training_data(jsonl_file):
    """Validate training data format and quality."""
    issues = []
    examples = []

    with open(jsonl_file, 'r') as f:
        for i, line in enumerate(f, 1):
            try:
                example = json.loads(line)
                examples.append(example)

                # Check required fields
                if 'messages' not in example:
                    issues.append(f"Line {i}: Missing 'messages' field")

                # Check message structure
                messages = example.get('messages', [])
                if len(messages) < 2:
                    issues.append(f"Line {i}: Need at least user + assistant messages")

                # Check for overly long examples
                total_length = sum(len(m['content']) for m in messages)
                if total_length > 4000:
                    issues.append(f"Line {i}: Example is very long ({total_length} chars)")

                # Check for duplicate examples
                # ... (implement duplicate detection)

            except json.JSONDecodeError:
                issues.append(f"Line {i}: Invalid JSON")

    print(f"Total examples: {len(examples)}")
    print(f"Issues found: {len(issues)}")
    for issue in issues[:10]:  # Show first 10 issues
        print(f"  - {issue}")

    return len(issues) == 0

# Validate before fine-tuning
is_valid = validate_training_data('training_data.jsonl')

import random

def split_data(input_file, train_ratio=0.9):
    """Split data into train and validation sets."""
    with open(input_file, 'r') as f:
        examples = [json.loads(line) for line in f]

    # Shuffle
    random.shuffle(examples)

    # Split
    split_idx = int(len(examples) * train_ratio)
    train_examples = examples[:split_idx]
    val_examples = examples[split_idx:]

    # Write to separate files
    with open('train.jsonl', 'w') as f:
        for ex in train_examples:
            f.write(json.dumps(ex) + '\n')

    with open('validation.jsonl', 'w') as f:
        for ex in val_examples:
            f.write(json.dumps(ex) + '\n')

    print(f"Training examples: {len(train_examples)}")
    print(f"Validation examples: {len(val_examples)}")

split_data('training_data.jsonl', train_ratio=0.9)

from openai import OpenAI
client = OpenAI()

# Upload training file
training_file = client.files.create(
    file=open("train.jsonl", "rb"),
    purpose="fine-tune"
)

print(f"Training file ID: {training_file.id}")

# Upload validation file (optional but recommended)
validation_file = client.files.create(
    file=open("validation.jsonl", "rb"),
    purpose="fine-tune"
)

print(f"Validation file ID: {validation_file.id}")

# Create fine-tuning job
fine_tune = client.fine_tuning.jobs.create(
    training_file=training_file.id,
    validation_file=validation_file.id,
    model="gpt-4o-mini-2024-07-18",
    hyperparameters={
        "n_epochs": 3,  # Number of training epochs (1-50, default: auto)
        "batch_size": "auto",  # Auto-select optimal batch size
        "learning_rate_multiplier": "auto"  # Auto-tune learning rate
    },
    suffix="customer-support-v1"  # Name your model
)

print(f"Fine-tuning job created: {fine_tune.id}")
print(f"Status: {fine_tune.status}")

import time

def monitor_fine_tuning(job_id):
    """Monitor fine-tuning progress."""
    while True:
        job = client.fine_tuning.jobs.retrieve(job_id)
        print(f"Status: {job.status}")

        if job.status == "succeeded":
            print(f"Fine-tuning complete!")
            print(f"Fine-tuned model: {job.fine_tuned_model}")
            return job.fine_tuned_model

        elif job.status == "failed":
            print(f"Fine-tuning failed: {job.error}")
            return None

        # Check training metrics if available
        if hasattr(job, 'result_files') and job.result_files:
            print("Training metrics available")

        time.sleep(60)  # Check every minute

# Monitor your job
model_name = monitor_fine_tuning(fine_tune.id)

# List training events (loss, accuracy over time)
events = client.fine_tuning.jobs.list_events(fine_tuning_job_id=fine_tune.id)

for event in events.data[:10]:
    print(event.message)

# Download result files for detailed analysis
results = client.fine_tuning.jobs.retrieve(fine_tune.id)
if results.result_files:
    for file_id in results.result_files:
        content = client.files.content(file_id)
        # Parse and analyze metrics
        print(content.text)

# Use the fine-tuned model exactly like base models
response = client.chat.completions.create(
    model=model_name,  # Your fine-tuned model ID
    messages=[
        {"role": "system", "content": "You are a helpful customer support assistant."},
        {"role": "user", "content": "I forgot my password, can you help?"}
    ]
)

print(response.choices[0].message.content)

# Install required libraries
pip install torch transformers datasets peft accelerate bitsandbytes

# For training on multiple GPUs
pip install deepspeed

from transformers import AutoModelForCausalLM, AutoTokenizer
from datasets import load_dataset
from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
import torch

# Load base model (e.g., Llama 2 7B, Mistral 7B)
model_name = "mistralai/Mistral-7B-v0.1"

# Load in 4-bit for memory efficiency (QLoRA)
from transformers import BitsAndBytesConfig

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.float16,
    bnb_4bit_use_double_quant=True
)

model = AutoModelForCausalLM.from_pretrained(
    model_name,
    quantization_config=bnb_config,
    device_map="auto",
    trust_remote_code=True
)

tokenizer = AutoTokenizer.from_pretrained(model_name)
tokenizer.pad_token = tokenizer.eos_token

# Prepare model for training
model = prepare_model_for_kbit_training(model)

# LoRA configuration
lora_config = LoraConfig(
    r=16,  # Rank of LoRA matrices (8, 16, 32, 64)
    lora_alpha=32,  # Scaling factor (typically 2x rank)
    target_modules=["q_proj", "k_proj", "v_proj", "o_proj"],  # Which layers to adapt
    lora_dropout=0.05,  # Dropout for regularization
    bias="none",  # Whether to train biases
    task_type="CAUSAL_LM"  # Task type
)

# Apply LoRA to model
model = get_peft_model(model, lora_config)
model.print_trainable_parameters()
# Output: "trainable params: 4,194,304 || all params: 7,241,748,480 || trainable%: 0.06%"

# Load your JSONL data
dataset = load_dataset("json", data_files={
    "train": "train.jsonl",
    "validation": "validation.jsonl"
})

def format_prompts(examples):
    """Format examples for instruction following."""
    formatted_texts = []

    for messages in examples["messages"]:
        text = ""
        for message in messages:
            if message["role"] == "system":
                text += f"<|system|>\n{message['content']}\n"
            elif message["role"] == "user":
                text += f"<|user|>\n{message['content']}\n"
            elif message["role"] == "assistant":
                text += f"<|assistant|>\n{message['content']}</s>\n"

        formatted_texts.append(text)

    return {"text": formatted_texts}

# Apply formatting
formatted_dataset = dataset.map(
    format_prompts,
    batched=True,
    remove_columns=dataset["train"].column_names
)

from transformers import TrainingArguments, Trainer

training_args = TrainingArguments(
    output_dir="./fine-tuned-model",
    num_train_epochs=3,
    per_device_train_batch_size=4,
    gradient_accumulation_steps=4,  # Effective batch size = 16
    learning_rate=2e-4,
    fp16=True,  # Mixed precision training
    logging_steps=10,
    save_steps=100,
    evaluation_strategy="steps",
    eval_steps=100,
    save_total_limit=2,  # Keep only 2 checkpoints
    load_best_model_at_end=True,
    warmup_steps=50,
    optim="paged_adamw_8bit",  # Memory-efficient optimizer
)

# Create trainer
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=formatted_dataset["train"],
    eval_dataset=formatted_dataset["validation"],
    tokenizer=tokenizer,
)

# Start training
trainer.train()

# Save LoRA adapter (small, ~100MB)
model.save_pretrained("./lora-adapter")
tokenizer.save_pretrained("./lora-adapter")

# Load and use later
from peft import PeftModel

base_model = AutoModelForCausalLM.from_pretrained(model_name)
fine_tuned_model = PeftModel.from_pretrained(base_model, "./lora-adapter")

# Generate with fine-tuned model
inputs = tokenizer("Your prompt here", return_tensors="pt")
outputs = fine_tuned_model.generate(**inputs, max_length=200)
print(tokenizer.decode(outputs[0]))

import json
from sklearn.metrics import accuracy_score, f1_score

def evaluate_model(model, tokenizer, test_file):
    """Evaluate model on test set."""
    predictions = []
    ground_truth = []

    with open(test_file, 'r') as f:
        for line in f:
            example = json.loads(line)
            messages = example["messages"]

            # Extract user message and expected response
            user_msg = next(m["content"] for m in messages if m["role"] == "user")
            expected = next(m["content"] for m in messages if m["role"] == "assistant")

            # Generate prediction
            prompt = f"User: {user_msg}\nAssistant:"
            inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
            outputs = model.generate(**inputs, max_length=500)
            prediction = tokenizer.decode(outputs[0], skip_special_tokens=True)

            predictions.append(prediction)
            ground_truth.append(expected)

    # For classification tasks
    # accuracy = accuracy_score(ground_truth, predictions)

    # For generation tasks, use other metrics
    from rouge import Rouge
    rouge = Rouge()
    scores = rouge.get_scores(predictions, ground_truth, avg=True)

    return {
        "rouge-1": scores["rouge-1"]["f"],
        "rouge-2": scores["rouge-2"]["f"],
        "rouge-l": scores["rouge-l"]["f"]
    }

# Compare models
base_scores = evaluate_model(base_model, tokenizer, "test.jsonl")
finetuned_scores = evaluate_model(finetuned_model, tokenizer, "test.jsonl")

print(f"Base model ROUGE-L: {base_scores['rouge-l']:.3f}")
print(f"Fine-tuned ROUGE-L: {finetuned_scores['rouge-l']:.3f}")
print(f"Improvement: {(finetuned_scores['rouge-l'] - base_scores['rouge-l']):.3f}")

def compare_outputs(base_model, finetuned_model, tokenizer, prompts):
    """Side-by-side comparison of model outputs."""
    for prompt in prompts:
        print(f"\nPrompt: {prompt}")
        print("-" * 80)

        # Base model
        base_output = generate_response(base_model, tokenizer, prompt)
        print(f"Base model:\n{base_output}")

        print()

        # Fine-tuned model
        ft_output = generate_response(finetuned_model, tokenizer, prompt)
        print(f"Fine-tuned model:\n{ft_output}")

        print("=" * 80)

# Test on diverse prompts
test_prompts = [
    "How do I reset my password?",
    "I was charged twice, need a refund",
    "Can you explain how your pricing works?",
    # Include edge cases
    "asdfjkl;",  # Gibberish
    "This product sucks!!!",  # Hostile
]

compare_outputs(base_model, finetuned_model, tokenizer, test_prompts)

import matplotlib.pyplot as plt

def plot_training_curves(log_history):
    """Plot training and validation loss."""
    train_loss = [x["loss"] for x in log_history if "loss" in x]
    eval_loss = [x["eval_loss"] for x in log_history if "eval_loss" in x]

    plt.figure(figsize=(10, 6))
    plt.plot(train_loss, label="Training Loss")
    plt.plot(eval_loss, label="Validation Loss")
    plt.xlabel("Steps")
    plt.ylabel("Loss")
    plt.legend()
    plt.title("Training Curves")
    plt.savefig("training_curves.png")

    # Check for overfitting
    final_train = train_loss[-1]
    final_val = eval_loss[-1]

    if final_val > final_train * 1.5:
        print("WARNING: Potential overfitting detected")
        print(f"Train loss: {final_train:.4f}, Val loss: {final_val:.4f}")

plot_training_curves(trainer.state.log_history)

response = client.chat.completions.create(
    model="ft:gpt-4o-mini-2024-07-18:company:model-name:abc123",
    messages=[...]
)

# Install vLLM for efficient inference
pip install vllm

# Merge LoRA adapter with base model (optional but faster)
from peft import PeftModel
merged_model = PeftModel.from_pretrained(base_model, "./lora-adapter")
merged_model = merged_model.merge_and_unload()
merged_model.save_pretrained("./merged-model")

# Start vLLM server
python -m vllm.entrypoints.openai.api_server \
    --model ./merged-model \
    --host 0.0.0.0 \
    --port 8000 \
    --tensor-parallel-size 1

# Use like OpenAI API
from openai import OpenAI
client = OpenAI(
    base_url="http://localhost:8000/v1",
    api_key="dummy"  # Not required for local
)

response = client.chat.completions.create(
    model="./merged-model",
    messages=[...]
)

# Push to Hugging Face Hub
model.push_to_hub("your-username/your-model-name")
tokenizer.push_to_hub("your-username/your-model-name")

# Deploy on Hugging Face Inference Endpoints
# Or use Replicate, Banana, etc. for auto-scaling