The Bias-Variance Tradeoff Explained

Expected Test Error = Bias² + Variance + Irreducible Noise

Bias²:             Error from wrong assumptions (model too simple)
Variance:          Error from sensitivity to training data (model too complex)
Irreducible Noise: Error in the labels themselves — can't be reduced

High Bias, Low Variance:     High Bias, High Variance:
   . . .                        .   .
  . X .   ← shots clustered       X       ← shots scattered
   . . .     away from center   .   .       and off-center
   
Consistent but wrong         Inconsistent and wrong
(underfitting)               (worst case)

Low Bias, High Variance:     Low Bias, Low Variance:
  .   .                           .
.   X   .  ← shots scattered     .X.    ← clustered on target
  .   .       around the center    .    
  
Inconsistent but centered    GOAL: what we want
(overfitting)                consistent and correct

import numpy as np
from sklearn.model_selection import cross_val_score
from sklearn.tree import DecisionTreeClassifier
from sklearn.linear_model import LogisticRegression

X = np.random.randn(500, 10)
y = (X[:, 0] ** 2 + X[:, 1] + np.random.randn(500) * 0.5 > 1).astype(int)

print(f"{'Depth':<8} {'Train Mean':<12} {'Val Mean':<10} {'Val Std':<10} {'Diagnosis'}")
print("-" * 60)

# As we increase complexity (depth), bias decreases then variance dominates
for depth in [1, 2, 3, 4, 5, 7, 10, None]:
    model = DecisionTreeClassifier(max_depth=depth, random_state=42)
    cv_scores = cross_val_score(model, X, y, cv=5, scoring="accuracy")
    model.fit(X[:400], y[:400])
    train_acc = model.score(X[:400], y[:400])

    depth_str = str(depth) if depth else "None"
    diagnosis = (
        "Underfitting" if cv_scores.mean() < 0.65 else
        "Good fit"     if cv_scores.std() < 0.04 and cv_scores.mean() > 0.70 else
        "Overfitting"  if train_acc - cv_scores.mean() > 0.10 else
        "OK"
    )

    print(f"{depth_str:<8} {train_acc:<12.3f} {cv_scores.mean():<10.3f} "
          f"{cv_scores.std():<10.3f} {diagnosis}")

# depth=1:    underfitting (high bias)
# depth=3-4:  best balance
# depth=None: overfitting (high variance)

Error
  │
  │  Training error:
  │  ╲_______________  (always decreases with complexity)
  │
  │  Test error:
  │  ╲__  ╱           (U-shape: decreases then increases)
  │     ╲╱
  │     ↑
  │  Sweet spot: optimal complexity
  │
  └─────────────────────────────────→ Model complexity
  
  Simple ←─────────────────────────→ Complex
  High bias, low variance           Low bias, high variance

from sklearn.model_selection import GridSearchCV, cross_val_score

# Strategy 1: Start simple, add complexity only if needed
models_by_complexity = [
    ("Logistic Regression (baseline)", LogisticRegression()),
    ("Decision Tree d=3",             DecisionTreeClassifier(max_depth=3)),
    ("Random Forest",                 RandomForestClassifier(n_estimators=100, max_depth=5)),
    ("Gradient Boosting",             GradientBoostingClassifier(max_depth=3)),
]

for name, model in models_by_complexity:
    cv = cross_val_score(model, X, y, cv=5, scoring="accuracy")
    print(f"{name}: {cv.mean():.3f} ± {cv.std():.3f}")

# Strategy 2: Regularization path — try many regularization strengths
for C in [0.001, 0.01, 0.1, 1, 10, 100]:
    cv = cross_val_score(LogisticRegression(C=C), X, y, cv=5, scoring="accuracy")
    print(f"C={C}: {cv.mean():.3f} ± {cv.std():.3f}")
# Watch: as C increases, std increases (more variance)

# Regularization is the primary tool for navigating the tradeoff
# More regularization → more bias, less variance
# Less regularization → less bias, more variance

from sklearn.linear_model import Ridge
from sklearn.datasets import make_regression

X, y = make_regression(n_samples=200, n_features=100, noise=20, random_state=42)

for alpha in [0.001, 0.01, 0.1, 1.0, 10.0, 100.0]:
    scores = cross_val_score(Ridge(alpha=alpha), X, y, cv=5, scoring="r2")
    print(f"alpha={alpha:6.3f}: R²={scores.mean():.3f} ± {scores.std():.3f}")

# alpha near 0: low bias, high variance (may overfit)
# alpha very high: high bias, low variance (underfits)
# Optimal alpha: highest mean R² with acceptable std