Recurrent Neural Networks (RNN)

# Simple RNN: Understanding Sequential Processing
import numpy as np
# Simple RNN from scratch
class SimpleRNN:
    def __init__(self, input_size, hidden_size, output_size):
        # Initialize weights
        self.W_xh = np.random.randn(input_size, hidden_size) * 0.01   # Input to hidden
        self.W_hh = np.random.randn(hidden_size, hidden_size) * 0.01  # Hidden to hidden
        self.W_hy = np.random.randn(hidden_size, output_size) * 0.01  # Hidden to output
        self.b_h = np.zeros((1, hidden_size))
        self.b_y = np.zeros((1, output_size))
    def forward(self, inputs):
        """
        Process sequence step by step
        inputs: list of input vectors (sequence)
        """
        h = np.zeros((1, self.W_hh.shape[0]))  # Initial hidden state
        outputs = []
        print(f"Processing sequence of length {len(inputs)}:")
        print(f"Initial hidden state: {h[0, :3].round(2)}...\n")
        # Process each time step
        for t, x in enumerate(inputs):
            # RNN core equation: h_t = tanh(W_xh·x_t + W_hh·h_{t-1} + b_h)
            h = np.tanh(x @ self.W_xh + h @ self.W_hh + self.b_h)
            # Output: y_t = W_hy·h_t + b_y
            y = h @ self.W_hy + self.b_y
            print(f"Step {t}: Input={x[0]:.2f}, Hidden state={h[0, :3].round(2)}..., Output={y[0, 0]:.4f}")
            outputs.append(y)
        return outputs, h
# Example: Process a simple sequence
print("=== Simple RNN Demo ===\n")
rnn = SimpleRNN(input_size=1, hidden_size=5, output_size=1)
# Input sequence: [1, 2, 3]
sequence = [np.array([[1.0]]), np.array([[2.0]]), np.array([[3.0]])]
outputs, final_hidden = rnn.forward(sequence)
print("\nKey Observation:")
print("- Hidden state changes at each step (memory!)")
print("- Each output depends on current input AND previous hidden state")
print("- Same weights (W_xh, W_hh, W_hy) used at all steps")
# Compare: What if inputs were independent? (feedforward)
print("\n=== Why RNN vs Feedforward? ===")
print("Feedforward: Each input processed independently, no memory")
print("RNN: Hidden state connects inputs, enabling context understanding")
print("\nExample: Understanding 'I like cats. They are cute.'")
print("- Feedforward: Can't connect 'They' to 'cats'")
print("- RNN: Hidden state remembers 'cats', understands 'They' refers to cats")

# RNN for Sequence Processing: Character-Level Language Model
import numpy as np
def softmax(x):
    exp_x = np.exp(x - np.max(x))
    return exp_x / np.sum(exp_x)
# Example: Learn to predict next character
text = "hello"
chars = list(set(text))
char_to_idx = {ch: i for i, ch in enumerate(chars)}
idx_to_char = {i: ch for ch, i in char_to_idx.items()}
vocab_size = len(chars)
hidden_size = 10
# Initialize RNN weights
W_xh = np.random.randn(vocab_size, hidden_size) * 0.01
W_hh = np.random.randn(hidden_size, hidden_size) * 0.01
W_hy = np.random.randn(hidden_size, vocab_size) * 0.01
b_h = np.zeros((hidden_size, 1))
b_y = np.zeros((vocab_size, 1))
def forward_pass(inputs, targets):
    """
    Forward pass through RNN
    inputs: list of character indices
    targets: list of target character indices (next char)
    """
    h = np.zeros((hidden_size, 1))  # Initial hidden state
    loss = 0
    print(f"\nProcessing: '{text}'")
    print(f"Goal: Predict next character at each step\n")
    for t in range(len(inputs)):
        # One-hot encode input
        x = np.zeros((vocab_size, 1))
        x[inputs[t]] = 1
        # RNN forward step
        h = np.tanh(W_xh.T @ x + W_hh @ h + b_h)
        # Output layer
        y = W_hy.T @ h + b_y
        probs = softmax(y)
        # Loss (cross-entropy)
        loss += -np.log(probs[targets[t], 0])
        # Display
        input_char = idx_to_char[inputs[t]]
        target_char = idx_to_char[targets[t]]
        predicted_idx = np.argmax(probs)
        predicted_char = idx_to_char[predicted_idx]
        print(f"Step {t}: Input='{input_char}' -> Target='{target_char}' | Predicted='{predicted_char}' (prob={probs[targets[t], 0]:.3f})")
    return loss / len(inputs)
# Prepare data: predict next character
inputs = [char_to_idx[ch] for ch in text[:-1]]   # "hell"
targets = [char_to_idx[ch] for ch in text[1:]]   # "ello"
loss = forward_pass(inputs, targets)
print(f"\nAverage Loss: {loss:.4f}")
print("\n=== Key Insights ===")
print("1. Hidden state h carries information from previous characters")
print("2. At each step, RNN predicts next character based on:")
print("   - Current input (current character)")
print("   - Hidden state (memory of previous characters)")
print("3. After training, RNN learns patterns: 'h' -> 'e', 'l' -> 'l', etc.")
print("4. Same weights used for all steps (parameter sharing)")
# Demonstrating memory
print("\n=== RNN Memory Demo ===")
print("Input sequence: h -> e -> l -> l")
print("\nHidden state evolution (first 3 values):")
h = np.zeros((hidden_size, 1))
for t, char in enumerate("hell"):
    x = np.zeros((vocab_size, 1))
    x[char_to_idx[char]] = 1
    h = np.tanh(W_xh.T @ x + W_hh @ h + b_h)
    print(f"After '{char}': h = [{h[0,0]:.3f}, {h[1,0]:.3f}, {h[2,0]:.3f}, ...]")
print("\nHidden state changes at each step, carrying sequence information!")