Tokenization in NLP

# Basic Tokenization Examples
import re
# 1. Simple Whitespace Tokenization
text = "Hello world! How are you?"
tokens = text.split()
print(tokens)
# Output: ['Hello', 'world!', 'How', 'are', 'you?']
# 2. Word Tokenization (with punctuation handling)
def word_tokenize(text):
    # Split on whitespace and punctuation
    return re.findall(r'\b\w+\b|[^\w\s]', text)
tokens = word_tokenize("Hello world! How are you?")
print(tokens)
# Output: ['Hello', 'world', '!', 'How', 'are', 'you', '?']
# 3. Character Tokenization
text = "Hello"
char_tokens = list(text)
print(char_tokens)
# Output: ['H', 'e', 'l', 'l', 'o']
# 4. Building Vocabulary
def build_vocab(texts):
    vocab = set()
    for text in texts:
        tokens = word_tokenize(text.lower())
        vocab.update(tokens)
    # Add special tokens
    vocab.update(['[PAD]', '[UNK]', '[CLS]', '[SEP]'])
    return {token: idx for idx, token in enumerate(sorted(vocab))}
texts = ["Hello world", "How are you", "Hello there"]
vocab = build_vocab(texts)
print(f"Vocabulary size: {len(vocab)}")
print(f"Sample mappings: {list(vocab.items())[:5]}")
# 5. Token to ID conversion
def tokens_to_ids(tokens, vocab):
    return [vocab.get(token, vocab['[UNK]']) for token in tokens]
tokens = word_tokenize("hello world")
token_ids = tokens_to_ids(tokens, vocab)
print(f"Tokens: {tokens}")
print(f"Token IDs: {token_ids}")

# Using Hugging Face Tokenizers - Industry Standard
from transformers import AutoTokenizer
# 1. BERT WordPiece Tokenizer
bert_tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')
text = "Tokenization is the first step in NLP."
encoded = bert_tokenizer(text, return_tensors='pt')
print("Tokens:", bert_tokenizer.tokenize(text))
print("Token IDs:", encoded['input_ids'])
print("Attention Mask:", encoded['attention_mask'])
# Decode back to text
decoded = bert_tokenizer.decode(encoded['input_ids'][0])
print("Decoded:", decoded)
# Output:
# Tokens: ['token', '##ization', 'is', 'the', 'first', 'step', 'in', 'nl', '##p', '.']
# Note: ## prefix indicates subword continuation
# 2. GPT-2 BPE Tokenizer
gpt2_tokenizer = AutoTokenizer.from_pretrained('gpt2')
text = "Tokenization is crucial for NLP models."
tokens = gpt2_tokenizer.tokenize(text)
print("GPT-2 Tokens:", tokens)
# BPE learns frequent subword patterns
# 3. Handling Special Tokens
text1 = "First sentence."
text2 = "Second sentence."
# For sentence pair tasks (e.g., question answering)
encoded = bert_tokenizer(
    text1,
    text2,
    padding='max_length',
    max_length=20,
    truncation=True,
    return_tensors='pt'
)
print("With special tokens:")
print(bert_tokenizer.convert_ids_to_tokens(encoded['input_ids'][0]))
# [CLS] first sentence . [SEP] second sentence . [SEP] [PAD] [PAD] ...
# 4. Batch Tokenization
texts = [
    "Short text",
    "This is a much longer text that needs to be tokenized",
    "Medium length"
]
encoded_batch = bert_tokenizer(
    texts,
    padding=True,  # Pad to longest in batch
    truncation=True,
    max_length=15,
    return_tensors='pt'
)
print("Batch token IDs shape:", encoded_batch['input_ids'].shape)
print("Attention masks shape:", encoded_batch['attention_mask'].shape)
# 5. Handling OOV Words with Subword Tokenization
text = "COVID-19 is a coronavirus disease."  # COVID-19 might be OOV
tokens = bert_tokenizer.tokenize(text)
print("Tokens with OOV handling:", tokens)
# Subword tokenization breaks unknown words into known subwords
# e.g., ["co", "##vid", "-", "19"] or similar

# Implementing Simple BPE from Scratch
from collections import Counter, defaultdict
import re
def get_stats(vocab):
    """Count frequency of adjacent pairs"""
    pairs = defaultdict(int)
    for word, freq in vocab.items():
        symbols = word.split()
        for i in range(len(symbols) - 1):
            pairs[(symbols[i], symbols[i+1])] += freq
    return pairs
def merge_vocab(pair, vocab):
    """Merge the most frequent pair in vocabulary"""
    new_vocab = {}
    bigram = ' '.join(pair)
    replacement = ''.join(pair)
    for word in vocab:
        new_word = word.replace(bigram, replacement)
        new_vocab[new_word] = vocab[word]
    return new_vocab
def learn_bpe(text, num_merges=10):
    """Learn BPE merges from text"""
    # Initialize vocabulary (character-level)
    vocab = Counter()
    for word in text.lower().split():
        vocab[' '.join(word) + ' </w>'] += 1
    print(f"Initial vocabulary size: {len(vocab)}")
    print(f"Sample: {list(vocab.items())[:3]}")
    # Learn merges
    merges = []
    for i in range(num_merges):
        pairs = get_stats(vocab)
        if not pairs:
            break
        best_pair = max(pairs, key=pairs.get)
        vocab = merge_vocab(best_pair, vocab)
        merges.append(best_pair)
        print(f"Merge {i+1}: {best_pair} (freq: {pairs[best_pair]})")
    return merges, vocab
# Example usage
text = """
low low low low low
lower lower lower
lowest lowest
"""
merges, final_vocab = learn_bpe(text, num_merges=5)
print("\nLearned merges:")
for i, merge in enumerate(merges):
    print(f"{i+1}. {merge}")
print("\nFinal vocabulary:")
for word, freq in list(final_vocab.items())[:10]:
    print(f"{word}: {freq}")
# Applying BPE to new word
def apply_bpe(word, merges):
    """Apply learned merges to tokenize a word"""
    word = ' '.join(word) + ' </w>'
    for merge in merges:
        bigram = ' '.join(merge)
        replacement = ''.join(merge)
        word = word.replace(bigram, replacement)
    return word.split()
new_word = "lower"
tokens = apply_bpe(new_word, merges)
print(f"\nTokenizing '{new_word}': {tokens}")

# Using SentencePiece - Language-Agnostic Tokenization
import sentencepiece as spm
# 1. Training SentencePiece model from scratch
# First, save training data to file
training_data = """
This is a sample text for training.
SentencePiece works on raw text without preprocessing.
It handles all languages uniformly including 日本語 and العربية.
"""
with open('training_data.txt', 'w', encoding='utf-8') as f:
    f.write(training_data)
# Train the model
spm.SentencePieceTrainer.train(
    input='training_data.txt',
    model_prefix='tokenizer',
    vocab_size=1000,
    model_type='bpe',  # or 'unigram'
    character_coverage=1.0,  # For multilingual, use 0.9995
    pad_id=0,
    unk_id=1,
    bos_id=2,
    eos_id=3
)
# 2. Loading and using trained model
sp = spm.SentencePieceProcessor()
sp.load('tokenizer.model')
text = "Hello world! This is SentencePiece."
# Tokenize to pieces
tokens = sp.encode_as_pieces(text)
print("Tokens:", tokens)
# Output: ['▁Hello', '▁world', '!', '▁This', '▁is', '▁Sent', 'ence', 'P', 'iece', '.']
# ▁ represents space
# Tokenize to IDs
ids = sp.encode_as_ids(text)
print("Token IDs:", ids)
# Decode back
decoded = sp.decode_ids(ids)
print("Decoded:", decoded)
# 3. Using pre-trained T5 tokenizer (uses SentencePiece)
from transformers import T5Tokenizer
t5_tokenizer = T5Tokenizer.from_pretrained('t5-small')
text = "translate English to German: Hello, how are you?"
tokens = t5_tokenizer.tokenize(text)
print("T5 Tokens:", tokens)
# T5 uses SentencePiece with special task prefixes
encoded = t5_tokenizer(text, return_tensors='pt')
print("Encoded shape:", encoded['input_ids'].shape)

# Practical Tokenization Pipeline
from transformers import AutoTokenizer
import torch
class TextProcessor:
    """Complete text processing pipeline"""
    def __init__(self, model_name='bert-base-uncased', max_length=128):
        self.tokenizer = AutoTokenizer.from_pretrained(model_name)
        self.max_length = max_length
    def preprocess(self, text):
        """Clean and normalize text"""
        # Remove extra whitespace
        text = ' '.join(text.split())
        # Lowercase (if using uncased model)
        text = text.lower()
        return text
    def tokenize_single(self, text, add_special_tokens=True):
        """Tokenize a single text"""
        text = self.preprocess(text)
        encoded = self.tokenizer(
            text,
            max_length=self.max_length,
            padding='max_length',
            truncation=True,
            return_tensors='pt',
            add_special_tokens=add_special_tokens
        )
        return {
            'input_ids': encoded['input_ids'],
            'attention_mask': encoded['attention_mask'],
            'tokens': self.tokenizer.convert_ids_to_tokens(encoded['input_ids'][0])
        }
    def tokenize_batch(self, texts):
        """Tokenize multiple texts efficiently"""
        texts = [self.preprocess(t) for t in texts]
        encoded = self.tokenizer(
            texts,
            max_length=self.max_length,
            padding='max_length',
            truncation=True,
            return_tensors='pt'
        )
        return encoded
    def decode(self, token_ids, skip_special_tokens=True):
        """Convert token IDs back to text"""
        return self.tokenizer.decode(
            token_ids,
            skip_special_tokens=skip_special_tokens
        )
    def get_vocab_size(self):
        """Get tokenizer vocabulary size"""
        return self.tokenizer.vocab_size
    def analyze_tokenization(self, text):
        """Analyze how text is tokenized"""
        result = self.tokenize_single(text, add_special_tokens=True)
        print(f"Original text: {text}")
        print(f"Tokens: {result['tokens']}")
        print(f"Token IDs: {result['input_ids'][0].tolist()}")
        print(f"Attention mask: {result['attention_mask'][0].tolist()}")
        print(f"Number of tokens: {result['attention_mask'][0].sum().item()}")
        return result
# Example usage
processor = TextProcessor('bert-base-uncased', max_length=64)
# Single text
text = "Natural language processing is fascinating!"
result = processor.analyze_tokenization(text)
# Batch processing
texts = [
    "Short text",
    "This is a longer text that demonstrates batch tokenization",
    "Another example"
]
batch = processor.tokenize_batch(texts)
print(f"\nBatch shape: {batch['input_ids'].shape}")
# Decoding
decoded = processor.decode(batch['input_ids'][1])
print(f"Decoded text: {decoded}")
print(f"\nVocabulary size: {processor.get_vocab_size()}")