Evaluation: The Metrics That Matter

import numpy as np
 
# Pretend test set: 200 crops, only 10 of them contain a ball (5%)
y_true = np.array([1] * 10 + [0] * 190)
 
# A lazy model that ALWAYS says "no ball"
lazy = np.zeros(200, dtype=int)
 
accuracy = np.mean(lazy == y_true)
print(f"Lazy model accuracy: {accuracy:.0%}")
print("Balls it found:", int(np.sum((lazy == 1) & (y_true == 1))), "of", int(y_true.sum()))

import numpy as np
 
 
def confusion(y_true, y_pred):
    tp = int(np.sum((y_pred == 1) & (y_true == 1)))      # said ball, was ball
    fp = int(np.sum((y_pred == 1) & (y_true == 0)))      # said ball, was NOT (false alarm)
    fn = int(np.sum((y_pred == 0) & (y_true == 1)))      # said no ball, was a ball (miss)
    tn = int(np.sum((y_pred == 0) & (y_true == 0)))      # said no ball, was not
    return tp, fp, fn, tn
 
 
def metrics(tp, fp, fn, tn):
    accuracy = (tp + tn) / (tp + fp + fn + tn)
    precision = tp / (tp + fp) if (tp + fp) else 0.0      # of the alarms, how many were real?
    recall = tp / (tp + fn) if (tp + fn) else 0.0         # of the real balls, how many did we catch?
    f1 = 2 * precision * recall / (precision + recall) if (precision + recall) else 0.0
    return accuracy, precision, recall, f1
 
 
y_true = np.array([1] * 10 + [0] * 190)
rng = np.random.default_rng(3)
y_prob = np.where(y_true == 1, rng.normal(0.7, 0.2, 200), rng.normal(0.25, 0.2, 200)).clip(0, 1)   # pretend model output
 
for threshold in (0.3, 0.5, 0.7):
    y_pred = (y_prob >= threshold).astype(int)
    tp, fp, fn, tn = confusion(y_true, y_pred)
    acc, prec, rec, f1 = metrics(tp, fp, fn, tn)
    print(f"threshold {threshold}: TP={tp} FP={fp} FN={fn} TN={tn} | accuracy {acc:.0%} precision {prec:.0%} recall {rec:.0%} F1 {f1:.2f}")

import numpy as np
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.metrics import classification_report, confusion_matrix, precision_recall_curve
 
# Replace these two lines with real test results (see the Keras block below)
rng = np.random.default_rng(3)
y_true = np.array([1] * 40 + [0] * 160)
y_prob = np.where(y_true == 1, rng.normal(0.7, 0.2, 200), rng.normal(0.3, 0.2, 200)).clip(0, 1)
 
THRESHOLD = 0.5
y_pred = (y_prob >= THRESHOLD).astype(int)
 
cm = confusion_matrix(y_true, y_pred)                    # rows = truth, columns = prediction
print(classification_report(y_true, y_pred, target_names=["no_ball", "ball"], digits=3))
 
fig, (a1, a2) = plt.subplots(1, 2, figsize=(9, 3.6))
sns.heatmap(cm, annot=True, fmt="d", cmap="Blues", cbar=False, ax=a1,
            xticklabels=["no_ball", "ball"], yticklabels=["no_ball", "ball"])
a1.set_xlabel("predicted")
a1.set_ylabel("truth")
a1.set_title(f"Confusion matrix (threshold {THRESHOLD})")
 
precision, recall, thresholds = precision_recall_curve(y_true, y_prob)
a2.plot(recall, precision)
a2.set_xlabel("recall")
a2.set_ylabel("precision")
a2.set_title("Precision-recall curve")
plt.tight_layout()
plt.savefig("evaluation.png", dpi=100)
 
# Which threshold gives the best F1?
f1 = 2 * precision[:-1] * recall[:-1] / np.maximum(precision[:-1] + recall[:-1], 1e-9)
best = int(np.argmax(f1))
print(f"Best F1 {f1[best]:.2f} at threshold {thresholds[best]:.2f}")

import numpy as np
from tensorflow import keras
 
model = keras.models.load_model("ball_classifier.keras")
 
test_ds = keras.utils.image_dataset_from_directory(
    "crops/test",
    class_names=["no_ball", "ball"],      # same order as training: no_ball = 0, ball = 1
    label_mode="binary",
    image_size=(64, 64),
    batch_size=32,
    shuffle=False,                         # keep the order so predictions line up with labels
)
 
y_true = np.concatenate([labels.numpy().ravel() for _, labels in test_ds]).astype(int)
y_prob = model.predict(test_ds, verbose=0).ravel()
print("test crops:", len(y_true), " balls:", int(y_true.sum()))