Loss Functions

# Mean Squared Error (MSE) Implementation
import numpy as np
def mse_loss(y_true, y_pred):
    """Calculate Mean Squared Error"""
    return np.mean((y_true - y_pred) ** 2)
def mse_gradient(y_true, y_pred):
    """Gradient of MSE w.r.t. predictions"""
    n = len(y_true)
    return 2 * (y_pred - y_true) / n
# Example: House price prediction
y_true = np.array([250, 300, 400, 350, 500])  # Actual prices ($1000s)
y_pred = np.array([240, 320, 390, 360, 480])  # Predicted prices
loss = mse_loss(y_true, y_pred)
gradient = mse_gradient(y_true, y_pred)
print("True values:", y_true)
print("Predictions:", y_pred)
print(f"\nMSE Loss: {loss:.2f}")
print("Gradient:", gradient)
# Why MSE? Penalizes large errors heavily
small_error = np.array([299])  # Error = 1
large_error = np.array([290])  # Error = 10
print(f"\nSmall error (1): MSE = {mse_loss(np.array([300]), small_error):.2f}")
print(f"Large error (10): MSE = {mse_loss(np.array([300]), large_error):.2f}")
# Large error is penalized 100x more (10² vs 1²)!
# Training Example: Linear Regression
X = np.array([1, 2, 3, 4, 5])
y = np.array([2, 4, 6, 8, 10])  # y = 2*x
# Initialize weight
w = 0.5
# Train using MSE
for epoch in range(10):
    y_pred = w * X
    loss = mse_loss(y, y_pred)
    gradient = 2 * np.mean(X * (y_pred - y))
    w = w - 0.1 * gradient  # Gradient descent step
    if epoch % 2 == 0:
        print(f"Epoch {epoch}: w={w:.3f}, Loss={loss:.2f}")
print(f"\nFinal weight: {w:.3f} (close to optimal w=2.0)")

# Binary Cross-Entropy Loss Implementation
import numpy as np
def binary_cross_entropy(y_true, y_pred, epsilon=1e-15):
    """
    Binary Cross-Entropy Loss
    y_true: actual labels (0 or 1)
    y_pred: predicted probabilities (0 to 1)
    epsilon: small value to prevent log(0)
    """
    # Clip predictions to prevent log(0)
    y_pred = np.clip(y_pred, epsilon, 1 - epsilon)
    loss = -np.mean(
        y_true * np.log(y_pred) +
        (1 - y_true) * np.log(1 - y_pred)
    )
    return loss
# Example: Email spam classification (0=not spam, 1=spam)
y_true = np.array([1, 0, 1, 1, 0])  # Actual labels
y_pred = np.array([0.9, 0.1, 0.8, 0.7, 0.2])  # Predicted probabilities
loss = binary_cross_entropy(y_true, y_pred)
print("True labels:", y_true)
print("Predictions:", y_pred)
print(f"Binary Cross-Entropy Loss: {loss:.4f}")
# Why Cross-Entropy? Heavily penalizes confident wrong predictions
print("\n=== Why Cross-Entropy? ===")
# Case 1: Correct prediction with high confidence
correct_confident = binary_cross_entropy(np.array([1]), np.array([0.99]))
print(f"Correct & confident (true=1, pred=0.99): Loss = {correct_confident:.4f}")
# Case 2: Correct prediction with low confidence
correct_uncertain = binary_cross_entropy(np.array([1]), np.array([0.51]))
print(f"Correct & uncertain (true=1, pred=0.51): Loss = {correct_uncertain:.4f}")
# Case 3: Wrong prediction with high confidence (BAD!)
wrong_confident = binary_cross_entropy(np.array([1]), np.array([0.01]))
print(f"Wrong & confident (true=1, pred=0.01): Loss = {wrong_confident:.4f}")
# Cross-entropy heavily penalizes confident wrong predictions!
# Full Example: Training Binary Classifier
from scipy.special import expit  # sigmoid function
# Data: Simple AND gate
X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]])
y = np.array([0, 0, 0, 1])
# Initialize weights
w = np.random.randn(2)
b = 0
learning_rate = 0.5
for epoch in range(100):
    # Forward pass
    z = X @ w + b
    y_pred = expit(z)  # Sigmoid activation
    # Compute loss
    loss = binary_cross_entropy(y, y_pred)
    # Backward pass (gradient)
    error = y_pred - y
    dw = X.T @ error / len(y)
    db = np.mean(error)
    # Update weights
    w -= learning_rate * dw
    b -= learning_rate * db
    if epoch % 20 == 0:
        print(f"Epoch {epoch}: Loss = {loss:.4f}")
print("\nFinal predictions:")
print(y_pred.round(2))  # Should be close to [0, 0, 0, 1]

# Categorical Cross-Entropy Loss Implementation
import numpy as np
def softmax(x):
    """Softmax activation function"""
    exp_x = np.exp(x - np.max(x, axis=-1, keepdims=True))
    return exp_x / np.sum(exp_x, axis=-1, keepdims=True)
def categorical_cross_entropy(y_true, y_pred, epsilon=1e-15):
    """
    Categorical Cross-Entropy Loss
    y_true: one-hot encoded labels (e.g., [0, 1, 0])
    y_pred: predicted probabilities for each class
    """
    y_pred = np.clip(y_pred, epsilon, 1 - epsilon)
    return -np.mean(np.sum(y_true * np.log(y_pred), axis=-1))
# Example: Image classification (3 classes: cat, dog, bird)
# True label: dog (class 1)
y_true = np.array([
    [0, 1, 0],  # Sample 1: dog
    [1, 0, 0],  # Sample 2: cat
    [0, 0, 1],  # Sample 3: bird
])
# Model predictions (after softmax)
y_pred = np.array([
    [0.1, 0.8, 0.1],  # Confident: dog (correct!)
    [0.7, 0.2, 0.1],  # Confident: cat (correct!)
    [0.3, 0.4, 0.3],  # Uncertain: hard to tell (wrong!)
])
loss = categorical_cross_entropy(y_true, y_pred)
print("True labels (one-hot):")
print(y_true)
print("\nPredictions (probabilities):")
print(y_pred)
print(f"\nCategorical Cross-Entropy Loss: {loss:.4f}")
# Why Categorical Cross-Entropy?
print("\n=== Analyzing Each Sample ===")
for i in range(len(y_true)):
    true_class = np.argmax(y_true[i])
    pred_class = np.argmax(y_pred[i])
    sample_loss = categorical_cross_entropy(
        y_true[i:i+1],
        y_pred[i:i+1]
    )
    print(f"Sample {i+1}: True={true_class}, Pred={pred_class}, Loss={sample_loss:.4f}")
# Full Training Example
print("\n=== Training Multi-Class Classifier ===")
# Toy dataset: 4 samples, 3 features, 3 classes
X = np.array([
    [1, 2, 3],
    [2, 3, 4],
    [3, 4, 5],
    [4, 5, 6]
])
y_true = np.array([
    [1, 0, 0],  # Class 0
    [0, 1, 0],  # Class 1
    [0, 0, 1],  # Class 2
    [0, 1, 0],  # Class 1
])
# Initialize weights (3 features -> 3 classes)
W = np.random.randn(3, 3) * 0.1
b = np.zeros(3)
learning_rate = 0.1
for epoch in range(100):
    # Forward pass
    logits = X @ W + b
    y_pred = softmax(logits)
    # Compute loss
    loss = categorical_cross_entropy(y_true, y_pred)
    # Backward pass
    error = y_pred - y_true
    dW = X.T @ error / len(y_true)
    db = np.mean(error, axis=0)
    # Update weights
    W -= learning_rate * dW
    b -= learning_rate * db
    if epoch % 20 == 0:
        predictions = np.argmax(y_pred, axis=1)
        actual = np.argmax(y_true, axis=1)
        accuracy = np.mean(predictions == actual)
        print(f"Epoch {epoch}: Loss = {loss:.4f}, Accuracy = {accuracy:.2%}")
print("\nFinal predictions:")
print(y_pred.round(3))

# Hinge Loss Implementation
import numpy as np
def hinge_loss(y_true, y_pred):
    """
    Hinge Loss for binary classification
    y_true: true labels (-1 or +1)
    y_pred: predicted scores (not probabilities!)
    """
    return np.mean(np.maximum(0, 1 - y_true * y_pred))
# Example: Binary classification with SVM-style labels
y_true = np.array([1, -1, 1, 1, -1])  # Labels: +1 or -1
y_pred = np.array([2.1, -1.5, 0.8, 3.0, -0.5])  # Raw scores
loss = hinge_loss(y_true, y_pred)
print("True labels:", y_true)
print("Predictions (scores):", y_pred)
print(f"Hinge Loss: {loss:.4f}")
# Understanding Hinge Loss
print("\n=== Understanding Hinge Loss ===")
samples = [
    (1, 2.0, "Correct & confident"),
    (1, 0.5, "Correct but not beyond margin"),
    (1, -0.5, "Wrong!"),
    (-1, -2.0, "Correct & confident"),
]
for y, pred, description in samples:
    loss = hinge_loss(np.array([y]), np.array([pred]))
    margin = y * pred
    print(f"{description:30s} | y={y:2d}, pred={pred:5.1f}, margin={margin:5.1f}, loss={loss:.2f}")
# Key insight: Hinge loss = 0 when margin > 1
# Forces correct predictions to be confident (beyond margin)
# Training Example: Simple Linear SVM
print("\n=== Training Linear SVM ===")
# Linearly separable data
X = np.array([
    [1, 2], [2, 3], [3, 3],  # Class +1
    [6, 5], [7, 8], [8, 7],  # Class -1
])
y = np.array([1, 1, 1, -1, -1, -1])
# Initialize weights
w = np.random.randn(2) * 0.1
b = 0
learning_rate = 0.01
lambda_reg = 0.01  # Regularization
for epoch in range(100):
    # Forward pass
    scores = X @ w + b
    # Compute hinge loss
    loss = hinge_loss(y, scores)
    # Add L2 regularization
    loss += lambda_reg * np.sum(w ** 2)
    # Backward pass (subgradient)
    margin = y * scores
    dloss = np.where(margin < 1, -y, 0)
    dw = (X.T @ dloss) / len(y) + 2 * lambda_reg * w
    db = np.mean(dloss)
    # Update weights
    w -= learning_rate * dw
    b -= learning_rate * db
    if epoch % 20 == 0:
        predictions = np.sign(scores)
        accuracy = np.mean(predictions == y)
        print(f"Epoch {epoch}: Loss = {loss:.4f}, Accuracy = {accuracy:.2%}")
print("\nFinal decision boundary: w =", w.round(3), ", b =", round(b, 3))
print("Predictions:", np.sign(X @ w + b))
print("Actual:", y)