Fine-Tuning LLMs with LoRA & QLoRA: Complete Production Guide

Master parameter-efficient fine-tuning — from theory to production deployment with LoRA, QLoRA, and advanced techniques.

Introduction

Fine-tuning adapts pre-trained LLMs to specific tasks. Full fine-tuning updates all parameters — expensive, requires massive compute. Parameter-Efficient Fine-Tuning (PEFT) updates <1% of parameters — fast, cheap, nearly same performance.

This guide covers LoRA (Low-Rank Adaptation) and QLoRA (Quantized LoRA) with production-ready code.

LoRA Theory

Original weights: W ∈ ℝ^(d×k)
LoRA adapters:    A ∈ ℝ^(d×r), B ∈ ℝ^(r×k) where r << min(d,k)
Forward pass:     h = Wx + BAx = (W + BA)x

Key insight: ΔW = BA has rank r, so we only train A and B.

# peft/lora_theory.py
import torch
import torch.nn as nn
from typing import Optional

class LoRALinear(nn.Module):
    """LoRA-adapted linear layer."""

    def __init__(
        self,
        in_features: int,
        out_features: int,
        r: int = 16,
        alpha: int = 32,
        dropout: float = 0.1,
        bias: bool = False
    ):
        super().__init__()
        self.in_features = in_features
        self.out_features = out_features
        self.r = r
        self.alpha = alpha
        self.scaling = alpha / r

        # Original frozen weights
        self.weight = nn.Parameter(torch.empty(out_features, in_features))
        self.weight.requires_grad = False

        # LoRA adapters
        self.lora_A = nn.Parameter(torch.empty(r, in_features))
        self.lora_B = nn.Parameter(torch.empty(out_features, r))
        self.dropout = nn.Dropout(dropout)

        if bias:
            self.bias = nn.Parameter(torch.empty(out_features))
        else:
            self.register_parameter('bias', None)

        self.reset_parameters()

    def reset_parameters(self):
        nn.init.kaiming_uniform_(self.weight, a=5**0.5)
        nn.init.kaiming_uniform_(self.lora_A, a=5**0.5)
        nn.init.zeros_(self.lora_B)
        if self.bias is not None:
            nn.init.zeros_(self.bias)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        # Base model forward (frozen)
        base_output = nn.functional.linear(x, self.weight, self.bias)

        # LoRA forward (trainable)
        lora_output = (self.dropout(x) @ self.lora_A.T @ self.lora_B.T) * self.scaling

        return base_output + lora_output

    def merge_weights(self):
        """Merge LoRA into base weights for inference."""
        if self.r > 0:
            delta_w = (self.lora_B @ self.lora_A) * self.scaling
            self.weight.data += delta_w
            self.lora_A.data.zero_()
            self.lora_B.data.zero_()

def apply_lora_to_model(
    model: nn.Module,
    target_modules: list = None,
    r: int = 16,
    alpha: int = 32,
    dropout: float = 0.1
) -> nn.Module:
    """Apply LoRA to target modules (e.g., q_proj, v_proj)."""
    if target_modules is None:
        target_modules = ["q_proj", "v_proj", "k_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]

    for name, module in model.named_modules():
        if any(target in name for target in target_modules) and isinstance(module, nn.Linear):
            parent_name = ".".join(name.split(".")[:-1])
            child_name = name.split(".")[-1]
            parent = model.get_submodule(parent_name) if parent_name else model

            lora_layer = LoRALinear(
                module.in_features,
                module.out_features,
                r=r,
                alpha=alpha,
                dropout=dropout,
                bias=module.bias is not None
            )
            lora_layer.weight.data = module.weight.data.clone()
            if module.bias is not None:
                lora_layer.bias.data = module.bias.data.clone()

            setattr(parent, child_name, lora_layer)

    return model

Part 2: QLoRA — 4-bit Quantization + LoRA

QLoRA combines 4-bit NF4 quantization with LoRA for massive memory savings.

# peft/qlora.py
import torch
from transformers import (
    AutoModelForCausalLM,
    AutoTokenizer,
    BitsAndBytesConfig
)
from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training

def load_qlora_model(
    model_id: str,
    r: int = 64,
    alpha: int = 16,
    dropout: float = 0.1,
    target_modules: list = None,
    device_map: str = "auto"
):
    """Load model with 4-bit quantization + LoRA."""

    # 4-bit NF4 quantization config
    bnb_config = BitsAndBytesConfig(
        load_in_4bit=True,
        bnb_4bit_quant_type="nf4",
        bnb_4bit_compute_dtype=torch.bfloat16,
        bnb_4bit_use_double_quant=True,
    )

    # Load base model
    model = AutoModelForCausalLM.from_pretrained(
        model_id,
        quantization_config=bnb_config,
        device_map=device_map,
        torch_dtype=torch.bfloat16,
        trust_remote_code=True,
    )

    # Prepare for k-bit training
    model = prepare_model_for_kbit_training(model, use_gradient_checkpointing=True)

    # LoRA config
    if target_modules is None:
        target_modules = ["q_proj", "v_proj", "k_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]

    lora_config = LoraConfig(
        r=r,
        lora_alpha=alpha,
        lora_dropout=dropout,
        target_modules=target_modules,
        bias="none",
        task_type="CAUSAL_LM",
        modules_to_save=None,
    )

    model = get_peft_model(model, lora_config)
    model.print_trainable_parameters()

    return model

NF4 Quantization Details

NormalFloat4 (NF4) — optimal for normally distributed weights

Standard int4:     [-8, -7, ..., 7] uniform spacing
NF4:               16 quantiles of N(0,1) — more precision near 0

Double Quantization: Quantize the quantization constants (saves 0.37 bits/param)

# peft/nf4_details.py
NF4_QUANTILES = torch.tensor([
    -1.0, -0.6961928, -0.525073, -0.394917, -0.284441, -0.184773,
    -0.091050, 0.0, 0.079580, 0.160930, 0.246112, 0.337915,
    0.440709, 0.562617, 0.722957, 1.0
])

def quantize_nf4(tensor: torch.Tensor) -> tuple:
    """Quantize to NF4 with double quantization."""
    # Block-wise quantization (block_size=64)
    block_size = 64
    original_shape = tensor.shape
    tensor = tensor.view(-1, block_size)

    # Compute absmax per block
    absmax = tensor.abs().max(dim=1, keepdim=True).values
    absmax = absmax.clamp(min=1e-8)

    # Normalize
    normalized = tensor / absmax

    # Quantize to NF4
    distances = (normalized.unsqueeze(-1) - NF4_QUANTILES.to(tensor.device)).abs()
    quantized = distances.argmin(dim=-1).to(torch.uint8)

    # Double quantization: quantize absmax
    absmax_quantized = quantize_absmax(absmax)

    return quantized, absmax_quantized, original_shape

def dequantize_nf4(quantized, absmax_quantized, shape):
    """Dequantize NF4 back to float."""
    # Implementation for inference
    pass

Part 3: Training Pipeline

# peft/training.py
from transformers import TrainingArguments, Trainer
from peft import PeftModel
from datasets import load_dataset
import torch

def create_training_args(
    output_dir: str,
    num_epochs: int = 3,
    batch_size: int = 4,
    gradient_accumulation: int = 4,
    learning_rate: float = 2e-4,
    warmup_ratio: float = 0.03,
    max_grad_norm: float = 0.3,
    logging_steps: int = 10,
    save_steps: int = 100,
    eval_steps: int = 100,
    fp16: bool = False,
    bf16: bool = True,
    gradient_checkpointing: bool = True,
    optim: str = "paged_adamw_32bit",
    lr_scheduler_type: str = "cosine",
) -> TrainingArguments:
    """Production-ready training arguments."""

    return TrainingArguments(
        output_dir=output_dir,
        num_train_epochs=num_epochs,
        per_device_train_batch_size=batch_size,
        per_device_eval_batch_size=batch_size,
        gradient_accumulation_steps=gradient_accumulation,
        learning_rate=learning_rate,
        warmup_ratio=warmup_ratio,
        max_grad_norm=max_grad_norm,
        logging_steps=logging_steps,
        save_steps=save_steps,
        eval_steps=eval_steps,
        evaluation_strategy="steps",
        save_strategy="steps",
        load_best_model_at_end=True,
        metric_for_best_model="eval_loss",
        greater_is_better=False,
        fp16=fp16,
        bf16=bf16,
        gradient_checkpointing=gradient_checkpointing,
        optim=optim,
        lr_scheduler_type=lr_scheduler_type,
        report_to="wandb",
        run_name=f"lora-{model_id.split('/')[-1]}",
        ddp_find_unused_parameters=False,
        remove_unused_columns=False,
    )

def format_dataset(example, tokenizer, max_length=2048):
    """Format for instruction tuning."""
    # Alpaca format
    prompt = f"""### Instruction:
{example['instruction']}

### Input:
{example.get('input', '')}

### Response:
{example['output']}"""

    tokenized = tokenizer(
        prompt,
        truncation=True,
        max_length=max_length,
        padding=False,
        return_tensors=None,
    )

    # Labels = input_ids for causal LM
    tokenized["labels"] = tokenized["input_ids"].copy()
    return tokenized

def train_lora(
    model,
    tokenizer,
    train_dataset,
    eval_dataset,
    training_args: TrainingArguments,
):
    """Train LoRA adapters."""

    trainer = Trainer(
        model=model,
        args=training_args,
        train_dataset=train_dataset,
        eval_dataset=eval_dataset,
        tokenizer=tokenizer,
        data_collator=lambda x: {
            "input_ids": torch.nn.utils.rnn.pad_sequence(
                [torch.tensor(i["input_ids"]) for i in x],
                batch_first=True,
                padding_value=tokenizer.pad_token_id
            ),
            "attention_mask": torch.nn.utils.rnn.pad_sequence(
                [torch.tensor(i["attention_mask"]) for i in x],
                batch_first=True,
                padding_value=0
            ),
            "labels": torch.nn.utils.rnn.pad_sequence(
                [torch.tensor(i["labels"]) for i in x],
                batch_first=True,
                padding_value=-100
            ),
        },
    )

    trainer.train()
    return trainer

# Usage
if __name__ == "__main__":
    model_id = "meta-llama/Meta-Llama-3-8B"

    model = load_qlora_model(model_id)
    tokenizer = AutoTokenizer.from_pretrained(model_id)
    tokenizer.pad_token = tokenizer.eos_token

    dataset = load_dataset("tatsu-lab/alpaca", split="train")
    dataset = dataset.map(lambda x: format_dataset(x, tokenizer))

    train_size = int(0.9 * len(dataset))
    train_dataset = dataset.select(range(train_size))
    eval_dataset = dataset.select(range(train_size, len(dataset)))

    args = create_training_args("./lora-llama3-8b-alpaca")
    trainer = train_lora(model, tokenizer, train_dataset, eval_dataset, args)

    # Save LoRA adapters only
    trainer.model.save_pretrained("./lora-adapters")
    tokenizer.save_pretrained("./lora-adapters")

Part 4: Advanced LoRA Variants

AdaLoRA — Adaptive Rank Allocation

# peft/adalora.py
from peft import AdaLoraConfig, get_peft_model

def create_adalora_model(model, target_modules=None):
    """AdaLoRA: dynamically allocates rank per layer."""

    config = AdaLoraConfig(
        init_r=12,
        target_r=8,
        beta1=0.85,
        beta2=0.85,
        tinit=200,
        tfinal=1000,
        deltaT=10,
        lora_alpha=32,
        lora_dropout=0.1,
        target_modules=target_modules or ["q_proj", "v_proj"],
        task_type="CAUSAL_LM",
    )

    return get_peft_model(model, config)

LoRA+ — Different Learning Rates

# peft/lora_plus.py
from torch.optim import AdamW

def create_lora_plus_optimizer(model, lr_A=1e-4, lr_B=1e-3, weight_decay=0.01):
    """LoRA+: higher LR for B matrix (closer to output)."""

    params_A = []
    params_B = []
    params_other = []

    for name, param in model.named_parameters():
        if "lora_A" in name:
            params_A.append(param)
        elif "lora_B" in name:
            params_B.append(param)
        else:
            params_other.append(param)

    return AdamW([
        {"params": params_A, "lr": lr_A, "weight_decay": weight_decay},
        {"params": params_B, "lr": lr_B, "weight_decay": weight_decay},
        {"params": params_other, "lr": lr_A, "weight_decay": weight_decay},
    ])

# Or use PEFT's built-in
from peft import LoraConfig
config = LoraConfig(
    lora_alpha=32,
    lora_dropout=0.1,
    target_modules=["q_proj", "v_proj"],
    use_lora_plus=True,  # Enables LoRA+
    lora_plus_scale=16,  # lr_B = lr_A * scale
)

DoRA — Weight-Decomposed LoRA

# peft/dora.py
from peft import LoraConfig

# DoRA decomposes weight into magnitude + direction
# magnitude: ||W||_2 (scalar per output neuron)
# direction: W / ||W||_2 (unit vector)
# LoRA adapts direction only

config = LoraConfig(
    r=64,
    lora_alpha=16,
    use_dora=True,  # Enable DoRA
    target_modules=["q_proj", "v_proj", "k_proj", "o_proj"],
    task_type="CAUSAL_LM",
)

Part 5: Multi-Task & Continual Learning

Multi-LoRA (Multiple Adapters)

# peft/multi_lora.py
from peft import PeftModel, LoraConfig

class MultiLoRAModel:
    """Switch between multiple LoRA adapters at inference."""

    def __init__(self, base_model, adapter_configs: dict):
        self.base_model = base_model
        self.adapters = {}
        self.active_adapter = None

        for name, config in adapter_configs.items():
            self.add_adapter(name, config)

    def add_adapter(self, name: str, config: LoraConfig):
        self.base_model.add_adapter(config, adapter_name=name)
        self.adapters[name] = config

    def set_adapter(self, name: str):
        self.base_model.set_adapter(name)
        self.active_adapter = name

    def disable_adapter(self):
        self.base_model.disable_adapter()
        self.active_adapter = None

    def merge_adapter(self, name: str):
        self.base_model.merge_adapter(name)

    def unmerge_adapter(self):
        self.base_model.unmerge_adapter()

# Usage
base_model = AutoModelForCausalLM.from_pretrained("meta-llama/Meta-Llama-3-8B")
multi_lora = MultiLoRAModel(base_model, {
    "coding": LoraConfig(r=32, target_modules=["q_proj", "v_proj"]),
    "chat": LoraConfig(r=16, target_modules=["q_proj", "v_proj"]),
    "sql": LoraConfig(r=8, target_modules=["q_proj", "v_proj"]),
})

multi_lora.set_adapter("coding")
output = multi_lora.base_model.generate(...)

multi_lora.set_adapter("chat")
output = multi_lora.base_model.generate(...)

Continual Learning with LoRA

# peft/continual.py
class ContinualLoRA:
    """Add new LoRA adapters for new tasks without forgetting."""

    def __init__(self, base_model):
        self.base_model = base_model
        self.task_adapters = {}

    def learn_task(self, task_name: str, train_data, config: LoraConfig):
        # Freeze existing adapters
        for name in self.task_adapters:
            self.base_model.set_adapter(name)
            for param in self.base_model.parameters():
                param.requires_grad = False

        # Add new adapter
        self.base_model.add_adapter(config, adapter_name=task_name)
        self.base_model.set_adapter(task_name)

        # Train only new adapter
        self._train_adapter(train_data)

        self.task_adapters[task_name] = config

    def infer(self, task_name: str, inputs):
        self.base_model.set_adapter(task_name)
        return self.base_model.generate(inputs)

    def merge_all(self):
        """Merge all adapters into base weights."""
        for name in self.task_adapters:
            self.base_model.merge_adapter(name)

Part 6: Inference & Deployment

Merging for Fast Inference

# peft/inference.py
from peft import PeftModel

def merge_and_save(base_model_id: str, adapter_path: str, output_path: str):
    """Merge LoRA into base model for deployment."""

    base_model = AutoModelForCausalLM.from_pretrained(
        base_model_id,
        torch_dtype=torch.float16,
        device_map="auto",
    )

    model = PeftModel.from_pretrained(base_model, adapter_path)
    model = model.merge_and_unload()  # Merges weights, removes PEFT

    model.save_pretrained(output_path, safe_serialization=True)
    tokenizer = AutoTokenizer.from_pretrained(base_model_id)
    tokenizer.save_pretrained(output_path)

# For QLoRA models — need to load in 4-bit first
def merge_qlora(base_model_id: str, adapter_path: str, output_path: str):
    bnb_config = BitsAndBytesConfig(
        load_in_4bit=True,
        bnb_4bit_quant_type="nf4",
        bnb_4bit_compute_dtype=torch.bfloat16,
    )

    base_model = AutoModelForCausalLM.from_pretrained(
        base_model_id,
        quantization_config=bnb_config,
        device_map="auto",
    )

    model = PeftModel.from_pretrained(base_model, adapter_path)
    model = model.merge_and_unload()

    # Save in float16 for deployment
    model.save_pretrained(output_path, safe_serialization=True)

vLLM / TGI Deployment

# peft/deployment.py
# vLLM supports LoRA adapters natively!
from vllm import LLM, SamplingParams
from vllm.lora.request import LoRARequest

def serve_with_lora(base_model_path: str, adapter_path: str):
    """Serve base model + LoRA with vLLM."""

    llm = LLM(
        model=base_model_path,
        enable_lora=True,
        max_lora_rank=64,
        max_loras=10,  # Max concurrent adapters
        gpu_memory_utilization=0.9,
    )

    sampling_params = SamplingParams(
        temperature=0.7,
        top_p=0.9,
        max_tokens=512,
    )

    # Load adapter
    lora_request = LoRARequest("my_adapter", 1, adapter_path)

    outputs = llm.generate(
        ["Write a Python function for fibonacci:"],
        sampling_params,
        lora_request=lora_request,
    )

    return outputs[0].outputs[0].text

# Multiple adapters simultaneously
def serve_multi_lora(base_model_path: str, adapters: dict):
    """Serve multiple LoRAs, select per-request."""
    llm = LLM(model=base_model_path, enable_lora=True, max_loras=len(adapters))

    lora_requests = {
        name: LoRARequest(name, i+1, path)
        for i, (name, path) in enumerate(adapters.items())
    }

    def generate(prompt: str, adapter_name: str):
        return llm.generate(
            [prompt],
            SamplingParams(temperature=0.7, max_tokens=512),
            lora_request=lora_requests[adapter_name],
        )[0].outputs[0].text

    return generate

Part 7: Hyperparameter Guide

LoRA Hyperparameters

Parameter	Small Model (7B)	Large Model (70B)	Notes
r (rank)	16-64	64-128	Higher = more capacity
alpha	16-32	16-32	Scaling factor
dropout	0.05-0.1	0.1-0.2	Regularization
target_modules	q,v	q,k,v,o,gate,up,down	More modules = more params
learning_rate	2e-4	1e-4	Lower for larger models

Optimal Configs by Use Case

# peft/configs.py
LORA_CONFIGS = {
    "general_chat": {
        "r": 32, "alpha": 16, "dropout": 0.1,
        "target_modules": ["q_proj", "v_proj", "k_proj", "o_proj"],
        "lr": 2e-4, "epochs": 3,
    },
    "coding": {
        "r": 64, "alpha": 16, "dropout": 0.05,
        "target_modules": ["q_proj", "v_proj", "k_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
        "lr": 1e-4, "epochs": 2,
    },
    "sql_generation": {
        "r": 16, "alpha": 32, "dropout": 0.1,
        "target_modules": ["q_proj", "v_proj"],
        "lr": 3e-4, "epochs": 5,
    },
    "classification": {
        "r": 8, "alpha": 16, "dropout": 0.1,
        "target_modules": ["q_proj", "v_proj"],
        "lr": 5e-4, "epochs": 10,
    },
    "QLoRA_general": {
        "r": 64, "alpha": 16, "dropout": 0.1,
        "target_modules": ["q_proj", "v_proj", "k_proj", "o_proj"],
        "lr": 2e-4, "epochs": 3,
        "quantization": "4bit_nf4",
    },
}

Part 8: Troubleshooting

Common Issues & Fixes

# peft/troubleshooting.py
TROUBLESHOOTING = {
    "loss_not_decreasing": {
        "causes": [
            "Learning rate too high → reduce to 1e-4",
            "Gradient accumulation too low → increase",
            "Frozen weights not actually frozen → check requires_grad",
            "Data format wrong → verify labels match input_ids",
        ],
        "debug": "Print trainable params, check gradient flow",
    },
    "overfitting": {
        "causes": [
            "Rank too high → reduce r to 8-16",
            "Dropout too low → increase to 0.1-0.2",
            "Epochs too many → add early stopping",
            "Dataset too small → use data augmentation",
        ],
    },
    "OOM": {
        "solutions": [
            "Use QLoRA 4-bit",
            "Reduce batch_size, increase gradient_accumulation",
            "Enable gradient_checkpointing",
            "Use paged_adamw_32bit optimizer",
            "Reduce max_length",
        ],
    },
    "degraded_performance": {
        "causes": [
            "Wrong target_modules for architecture",
            "Alpha/r ratio off → try alpha=2*r",
            "Base model already good at task → less LoRA needed",
            "Merged weights incorrectly → verify merge_and_unload",
        ],
    },
    "inference_slow": {
        "solutions": [
            "Merge adapters before deployment",
            "Use vLLM with LoRA support",
            "Quantize merged model (AWQ/GPTQ)",
            "Reduce r for faster matmul",
        ],
    },
}

Debugging Checklist

# peft/debug.py
def debug_lora_setup(model):
    """Comprehensive LoRA debugging."""

    print("=== Trainable Parameters ===")
    trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
    total = sum(p.numel() for p in model.parameters())
    print(f"Trainable: {trainable:,} ({100*trainable/total:.2f}%)")
    print(f"Total: {total:,}")

    print("\n=== LoRA Modules ===")
    for name, module in model.named_modules():
        if hasattr(module, 'lora_A'):
            print(f"  {name}: r={module.r}, alpha={module.alpha}")
            print(f"    A: {module.lora_A.shape}, B: {module.lora_B.shape}")
            print(f"    A grad: {module.lora_A.grad is not None}")
            print(f"    B grad: {module.lora_B.grad is not None}")

    print("\n=== Gradient Flow Test ===")
    dummy_input = torch.randn(1, 10, 4096).to(next(model.parameters()).device)
    output = model(dummy_input)
    loss = output.logits.sum()
    loss.backward()

    has_grad = any(p.grad is not None and p.grad.abs().sum() > 0 
                   for p in model.parameters() if p.requires_grad)
    print(f"Gradients flowing: {has_grad}")

    if not has_grad:
        print("ERROR: No gradients! Check requires_grad and forward pass.")

Part 9: Evaluation & Benchmarking

# peft/evaluation.py
from datasets import load_dataset
from transformers import pipeline
import evaluate

def evaluate_lora(model, tokenizer, benchmark="mt_bench"):
    """Evaluate LoRA model on standard benchmarks."""

    if benchmark == "mt_bench":
        # MT-Bench: multi-turn conversation evaluation
        pass
    elif benchmark == "alpaca_eval":
        dataset = load_dataset("tatsu-lab/alpaca_eval", split="eval")
        # Generate responses
        pass
    elif benchmark == "mmlu":
        dataset = load_dataset("cais/mmlu", "all", split="test")
        # Multiple choice evaluation
        pass
    elif benchmark == "humaneval":
        dataset = load_dataset("openai_humaneval", split="test")
        # Code generation
        pass

def compare_adapters(model, tokenizer, adapters: dict, test_prompts: list):
    """Compare multiple LoRA adapters side-by-side."""

    results = {}
    for name, adapter_path in adapters.items():
        model.load_adapter(adapter_path, adapter_name=name)
        model.set_adapter(name)

        responses = []
        for prompt in test_prompts:
            inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
            with torch.no_grad():
                outputs = model.generate(**inputs, max_new_tokens=256, temperature=0.7)
            response = tokenizer.decode(outputs[0][inputs.input_ids.shape[1]:], skip_special_tokens=True)
            responses.append(response)

        results[name] = responses
        model.disable_adapter()

    return results

Part 10: Production Checklist

# peft/production_checklist.py
PRODUCTION_CHECKLIST = {
    "pre_training": [
        "✅ Validate dataset quality & format",
        "✅ Check tokenization (no truncation issues)",
        "✅ Verify base model license allows fine-tuning",
        "✅ Set random seeds for reproducibility",
        "✅ Configure W&B / MLflow tracking",
    ],
    "training": [
        "✅ Monitor train/val loss curves",
        "✅ Check gradient norms (should be ~1)",
        "✅ Verify learning rate schedule",
        "✅ Save checkpoints every N steps",
        "✅ Log GPU memory usage",
    ],
    "post_training": [
        "✅ Merge adapters for deployment",
        "✅ Run eval on holdout test set",
        "✅ Compare with base model (A/B test)",
        "✅ Test edge cases & safety",
        "✅ Benchmark latency/throughput",
    ],
    "deployment": [
        "✅ Quantize merged model (AWQ/GPTQ)",
        "✅ Test with vLLM / TGI",
        "✅ Configure autoscaling",
        "✅ Set up monitoring (latency, errors, quality)",
        "✅ Plan rollback strategy",
    ],
    "maintenance": [
        "✅ Schedule periodic re-training",
        "✅ Monitor data drift",
        "✅ Track model performance over time",
        "✅ Version adapters with git tags",
    ],
}

Resources

• LoRA Paper
• QLoRA Paper
• AdaLoRA Paper
• DoRA Paper
• PEFT Documentation
• BitsAndBytes
• vLLM LoRA Support

---

LoRA/QLoRA makes fine-tuning accessible. Start small (r=16), validate on your task, then scale. The best adapter is the one that ships.