The Test Set: One Shot, Final Score

from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score, classification_report
import numpy as np

X, y = np.random.randn(2000, 20), np.random.randint(0, 2, 2000)

# 1. Split data — test is locked away immediately
X_trainval, X_test, y_trainval, y_test = train_test_split(
    X, y, test_size=0.15, random_state=42, stratify=y
)

# 2. All development uses only trainval
X_train, X_val, y_train, y_val = train_test_split(X_trainval, y_trainval, test_size=0.18)

# 3. Train and tune (many iterations allowed here)
model = GradientBoostingClassifier(n_estimators=200, max_depth=3, learning_rate=0.05)
model.fit(X_train, y_train)

val_auc = roc_auc_score(y_val, model.predict_proba(X_val)[:, 1])
print(f"Validation AUC: {val_auc:.3f}")

# 4. Final evaluation — run this ONCE, after all decisions are made
test_auc = roc_auc_score(y_test, model.predict_proba(X_test)[:, 1])
print(f"Test AUC: {test_auc:.3f}")   # This is the number you report

print("\nClassification Report on Test Set:")
print(classification_report(y_test, model.predict(X_test)))

# WRONG: using test set to pick the best model
for model in [model_a, model_b, model_c]:
    test_score = model.score(X_test, y_test)   # Never do this during selection
    if test_score > best:
        best = test_score
        best_model = model

# WRONG: fitting preprocessing on all data (including test)
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
X_all_normalized = scaler.fit_transform(X)   # X includes test data!
X_train = X_all_normalized[:1700]
X_test  = X_all_normalized[1700:]            # Test statistics leaked into normalization

from sklearn.metrics import (
    roc_auc_score, average_precision_score,
    f1_score, classification_report, confusion_matrix
)

# Compute and report relevant metrics on test set
y_pred = model.predict(X_test)
y_proba = model.predict_proba(X_test)[:, 1]

report = {
    "test_auc_roc":            roc_auc_score(y_test, y_proba),
    "test_auc_pr":             average_precision_score(y_test, y_proba),
    "test_f1_macro":           f1_score(y_test, y_pred, average="macro"),
    "test_samples":            len(y_test),
    "positive_class_fraction": y_test.mean(),
}

for k, v in report.items():
    print(f"{k}: {v:.4f}" if isinstance(v, float) else f"{k}: {v}")

gap = val_auc - test_auc

if gap > 0.10:
    print("WARNING: Significant val/test gap. Investigate before deployment:")
    print("  1. Was the test set used previously for any decision?")
    print("  2. Are train/val/test from different time periods or sources?")
    print("  3. Is the test set large enough for a stable estimate?")

from sklearn.utils import resample

def bootstrap_auc(y_true, y_pred_proba, n_iterations: int = 1000, ci: float = 0.95):
    """95% confidence interval for AUC-ROC using bootstrap."""
    scores = []
    for _ in range(n_iterations):
        idx = resample(range(len(y_true)))
        score = roc_auc_score(y_true[idx], y_pred_proba[idx])
        scores.append(score)

    scores.sort()
    lo = int((1 - ci) / 2 * n_iterations)
    hi = int((1 + ci) / 2 * n_iterations)
    return scores[lo], np.mean(scores), scores[hi]

lo, mean, hi = bootstrap_auc(np.array(y_test), y_proba)
print(f"AUC-ROC: {mean:.3f} (95% CI: [{lo:.3f}, {hi:.3f}])")

# Production model evaluation is like a continuous test set
# New data arriving daily = fresh test examples
# Monitor performance over time for drift

class ProductionMonitor:
    def __init__(self, model, baseline_auc: float):
        self.model = model
        self.baseline_auc = baseline_auc
        self.recent_scores = []

    def evaluate_batch(self, X_new, y_new):
        """Evaluate model on a new batch — acts like a rolling test set."""
        score = roc_auc_score(y_new, self.model.predict_proba(X_new)[:, 1])
        self.recent_scores.append(score)

        if len(self.recent_scores) >= 30:
            rolling_auc = np.mean(self.recent_scores[-30:])
            if self.baseline_auc - rolling_auc > 0.05:
                print(f"ALERT: AUC dropped from {self.baseline_auc:.3f} to {rolling_auc:.3f}")