Word Embeddings

# Word Embeddings Basics with Gensim
from gensim.models import Word2Vec
import numpy as np
# Sample corpus
sentences = [
    ['king', 'queen', 'man', 'woman'],
    ['paris', 'france', 'berlin', 'germany'],
    ['cat', 'dog', 'animal', 'pet'],
    ['good', 'bad', 'happy', 'sad']
]
# Train Word2Vec model (Skip-gram)
model = Word2Vec(
    sentences,
    vector_size=100,  # Embedding dimension
    window=5,          # Context window
    min_count=1,       # Minimum word frequency
    workers=4,
    sg=1              # Skip-gram (1) or CBOW (0)
)
# Get vector for a word
king_vector = model.wv['king']
print(f"King vector shape: {king_vector.shape}")
print(f"King vector (first 5 dims): {king_vector[:5]}")
# Find similar words
similar_to_king = model.wv.most_similar('king', topn=3)
print(f"Words similar to 'king': {similar_to_king}")
# Word arithmetic: king - man + woman ≈ queen
result = model.wv.most_similar(
    positive=['king', 'woman'],
    negative=['man'],
    topn=1
)
print(f"king - man + woman = {result}")
# Cosine similarity
similarity = model.wv.similarity('king', 'queen')
print(f"Similarity(king, queen) = {similarity:.3f}")

# Using Pre-trained GloVe Embeddings
import numpy as np
from gensim.scripts.glove2word2vec import glove2word2vec
from gensim.models import KeyedVectors
# Convert GloVe format to Word2Vec format (one-time)
glove_file = 'glove.6B.100d.txt'  # Download from Stanford NLP
word2vec_file = 'glove.6B.100d.word2vec.txt'
glove2word2vec(glove_file, word2vec_file)
# Load pre-trained embeddings
embeddings = KeyedVectors.load_word2vec_format(word2vec_file, binary=False)
print(f"Vocabulary size: {len(embeddings)}")
print(f"Embedding dimension: {embeddings.vector_size}")
# Get embedding for a word
word = 'computer'
vector = embeddings[word]
print(f"{word} embedding: {vector[:5]}...")
# Find similar words
similar_words = embeddings.most_similar('computer', topn=5)
print(f"Words similar to '{word}': {similar_words}")
# Analogies: paris - france + germany ≈ berlin
result = embeddings.most_similar(
    positive=['paris', 'germany'],
    negative=['france'],
    topn=1
)
print(f"paris - france + germany = {result[0][0]}")
# Calculate similarity
sim = embeddings.similarity('car', 'automobile')
print(f"Similarity(car, automobile) = {sim:.3f}")

# Contextual Embeddings with BERT
from transformers import BertTokenizer, BertModel
import torch
# Load pre-trained BERT
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = BertModel.from_pretrained('bert-base-uncased')
# Two sentences with word "bank" in different contexts
sentence1 = "I went to the bank to deposit money"
sentence2 = "I sat on the river bank"
def get_word_embedding(sentence, word, tokenizer, model):
    # Tokenize
    tokens = tokenizer.tokenize(sentence)
    indexed_tokens = tokenizer.convert_tokens_to_ids(tokens)
    tokens_tensor = torch.tensor([indexed_tokens])
    # Get embeddings
    with torch.no_grad():
        outputs = model(tokens_tensor)
        # Last hidden state: (batch_size, sequence_length, hidden_size)
        hidden_states = outputs.last_hidden_state
    # Find word position
    word_idx = tokens.index(word)
    word_embedding = hidden_states[0, word_idx, :]
    return word_embedding.numpy()
# Get contextual embeddings for "bank" in both sentences
bank_embedding1 = get_word_embedding(sentence1, "bank", tokenizer, model)
bank_embedding2 = get_word_embedding(sentence2, "bank", tokenizer, model)
print(f"Bank embedding 1 shape: {bank_embedding1.shape}")  # (768,)
print(f"Bank embedding 2 shape: {bank_embedding2.shape}")  # (768,)
# Calculate cosine similarity
from numpy.linalg import norm
cosine_sim = np.dot(bank_embedding1, bank_embedding2) / (norm(bank_embedding1) * norm(bank_embedding2))
print(f"Cosine similarity between 'bank' in different contexts: {cosine_sim:.3f}")
# Lower similarity shows BERT captures different meanings!
print(f"Bank (financial) first 5 dims: {bank_embedding1[:5]}")
print(f"Bank (river) first 5 dims: {bank_embedding2[:5]}")