Gradient Descent

# Simple Gradient Descent Example: Finding Minimum of y = x²
import numpy as np
import matplotlib.pyplot as plt
# Function: y = x² (parabola with minimum at x=0)
def f(x):
    return x ** 2
# Gradient (derivative): dy/dx = 2x
def gradient(x):
    return 2 * x
# Gradient Descent Algorithm
def gradient_descent(start_x, learning_rate, num_iterations):
    x = start_x
    history = [x]
    for i in range(num_iterations):
        grad = gradient(x)  # Calculate gradient
        x = x - learning_rate * grad  # Update: move opposite to gradient
        history.append(x)
        print(f"Iteration {i+1}: x = {x:.4f}, f(x) = {f(x):.4f}, gradient = {grad:.4f}")
    return x, history
# Start from x=10, try to reach minimum at x=0
start_x = 10
learning_rate = 0.1
num_iterations = 10
final_x, history = gradient_descent(start_x, learning_rate, num_iterations)
print(f"\nStarted at x = {start_x}, f(x) = {f(start_x)}")
print(f"Ended at x = {final_x:.4f}, f(x) = {f(final_x):.4f}")
print("Found minimum! (x ≈ 0)")
# Output shows x moving closer to 0 (the minimum) each iteration:
# Iteration 1: x = 8.0000, f(x) = 64.0000, gradient = 20.0000
# Iteration 2: x = 6.4000, f(x) = 40.9600, gradient = 16.0000
# Iteration 3: x = 5.1200, f(x) = 26.2144, gradient = 12.8000
# ...
# The algorithm found the minimum by following the negative gradient!

# Complete Gradient Descent: Linear Regression Example
import numpy as np
# Dataset: Hours studied vs Exam Score
X = np.array([1, 2, 3, 4, 5])  # Hours studied
y = np.array([2, 4, 5, 4, 5])  # Exam score
# Model: y_pred = mx + b (linear regression)
# Initialize parameters randomly
m = 0.0  # slope
b = 0.0  # intercept
# Hyperparameters
learning_rate = 0.01
epochs = 1000
n = len(X)
# Training using Gradient Descent
for epoch in range(epochs):
    # 1. Make predictions
    y_pred = m * X + b
    # 2. Calculate loss (Mean Squared Error)
    loss = (1/n) * np.sum((y - y_pred) ** 2)
    # 3. Calculate gradients
    dm = -(2/n) * np.sum(X * (y - y_pred))  # ∂Loss/∂m
    db = -(2/n) * np.sum(y - y_pred)        # ∂Loss/∂b
    # 4. Update parameters (Gradient Descent step!)
    m = m - learning_rate * dm
    b = b - learning_rate * db
    # Print progress
    if epoch % 100 == 0:
        print(f"Epoch {epoch}: Loss = {loss:.4f}, m = {m:.4f}, b = {b:.4f}")
print(f"\nFinal Model: y = {m:.2f}x + {b:.2f}")
print(f"Prediction for 6 hours studied: {m * 6 + b:.2f}")
# Output:
# Epoch 0: Loss = 10.0000, m = 0.4000, b = 0.2800
# Epoch 100: Loss = 0.6863, m = 0.5918, b = 1.6294
# ...
# Epoch 900: Loss = 0.4714, m = 0.6000, b = 2.2000
#
# Final Model: y = 0.60x + 2.20
# Prediction for 6 hours studied: 5.80
# Gradient Descent found the best fit line by minimizing loss!

# Comparing Three Types of Gradient Descent
import numpy as np
X = np.array([[1], [2], [3], [4], [5]])
y = np.array([2, 4, 5, 4, 5])
def compute_gradient(X, y, theta):
    m = len(y)
    predictions = X.dot(theta)
    errors = predictions - y
    gradient = (1/m) * X.T.dot(errors)
    return gradient
# 1. BATCH GRADIENT DESCENT
def batch_gd(X, y, learning_rate=0.01, epochs=100):
    theta = np.zeros(X.shape[1])
    for epoch in range(epochs):
        gradient = compute_gradient(X, y, theta)  # Use ALL data
        theta = theta - learning_rate * gradient
    return theta
# 2. STOCHASTIC GRADIENT DESCENT
def stochastic_gd(X, y, learning_rate=0.01, epochs=100):
    theta = np.zeros(X.shape[1])
    for epoch in range(epochs):
        for i in range(len(X)):
            xi = X[i:i+1]  # Use ONE sample
            yi = y[i:i+1]
            gradient = compute_gradient(xi, yi, theta)
            theta = theta - learning_rate * gradient
    return theta
# 3. MINI-BATCH GRADIENT DESCENT
def mini_batch_gd(X, y, batch_size=2, learning_rate=0.01, epochs=100):
    theta = np.zeros(X.shape[1])
    for epoch in range(epochs):
        indices = np.random.permutation(len(X))
        X_shuffled = X[indices]
        y_shuffled = y[indices]
        for i in range(0, len(X), batch_size):
            Xi = X_shuffled[i:i+batch_size]  # Use BATCH of samples
            yi = y_shuffled[i:i+batch_size]
            gradient = compute_gradient(Xi, yi, theta)
            theta = theta - learning_rate * gradient
    return theta
# Compare
print("Batch GD:", batch_gd(X, y))
print("Stochastic GD:", stochastic_gd(X, y))
print("Mini-Batch GD:", mini_batch_gd(X, y))
# All converge to similar solutions, but with different speeds and stability!