Large Language Models (LLMs)

import tiktoken  # OpenAI's tokenizer

# Initialize GPT-4 tokenizer
enc = tiktoken.encoding_for_model("gpt-4")

# Example tokenization
text = "The quick brown fox jumps over the lazy dog"
tokens = enc.encode(text)
print(f"Text: {text}")
print(f"Tokens: {tokens}")
print(f"Decoded: {[enc.decode([t]) for t in tokens]}")

# Output:
# Tokens: [791, 4062, 14198, 39935, 35308, 927, 279, 16053, 5679]
# Decoded: ['The', ' quick', ' brown', ' fox', ' jumps', ' over', ' the', ' lazy', ' dog']

# Interesting cases
examples = [
    "artificial intelligence",  # Common phrase
    "🤖",                       # Emoji
    "GPT-4",                    # Model name
    "antidisestablishmentarianism"  # Long word
]

for text in examples:
    tokens = enc.encode(text)
    print(f"\n'{text}'")
    print(f"  Tokens: {len(tokens)}")
    print(f"  Breakdown: {[enc.decode([t]) for t in tokens]}")

# Output:
# 'artificial intelligence'
#   Tokens: 2
#   Breakdown: ['artificial', ' intelligence']
#
# '🤖'
#   Tokens: 1
#   Breakdown: ['🤖']
#
# 'GPT-4'
#   Tokens: 3
#   Breakdown: ['G', 'PT', '-4']
#
# 'antidisestablishmentarianism'
#   Tokens: 6
#   Breakdown: ['ant', 'idis', 'establish', 'ment', 'arian', 'ism']

import torch
import torch.nn as nn
from torch.optim import AdamW
from torch.nn.parallel import DistributedDataParallel as DDP

class LLMTrainer:
    def __init__(self, model, config):
        self.model = DDP(model)  # Distributed training across GPUs
        self.optimizer = AdamW(
            model.parameters(),
            lr=config.learning_rate,
            betas=(0.9, 0.95),
            weight_decay=0.1
        )
        self.scheduler = CosineAnnealingLR(self.optimizer)
        
    def train_step(self, batch):
        # Forward pass
        input_ids = batch['input_ids']  # [batch_size, seq_length]
        attention_mask = batch['attention_mask']
        
        # Shift targets for next-token prediction
        labels = input_ids[:, 1:].contiguous()
        input_ids = input_ids[:, :-1].contiguous()
        
        # Model forward
        outputs = self.model(
            input_ids=input_ids,
            attention_mask=attention_mask[:, :-1]
        )
        
        # Calculate loss (cross-entropy)
        logits = outputs.logits  # [batch_size, seq_length-1, vocab_size]
        loss = nn.CrossEntropyLoss()(
            logits.view(-1, logits.size(-1)),
            labels.view(-1)
        )
        
        # Backward pass
        loss.backward()
        
        # Gradient clipping to prevent explosion
        torch.nn.utils.clip_grad_norm_(self.model.parameters(), 1.0)
        
        # Optimizer step
        self.optimizer.step()
        self.optimizer.zero_grad()
        self.scheduler.step()
        
        return loss.item()

# Training configuration for a 7B model
config = {
    'batch_size': 4096,          # Large batch for stability
    'sequence_length': 2048,      # Context window
    'learning_rate': 6e-4,        # Peak learning rate
    'warmup_steps': 2000,         # Learning rate warmup
    'total_steps': 300000,        # Total training steps
    'gradient_accumulation': 16,  # Effective batch = 65K tokens
    'num_gpus': 128,             # A100 80GB GPUs
    'precision': 'bfloat16',     # Mixed precision training
}

import openai
from anthropic import Anthropic
import google.generativeai as genai

# OpenAI GPT-4
openai.api_key = "your-key"

def call_gpt4(prompt, temperature=0.7, max_tokens=1000):
    response = openai.ChatCompletion.create(
        model="gpt-4",
        messages=[
            {"role": "system", "content": "You are a helpful assistant."},
            {"role": "user", "content": prompt}
        ],
        temperature=temperature,  # 0=deterministic, 2=creative
        max_tokens=max_tokens,
        top_p=0.9,  # Nucleus sampling
        frequency_penalty=0.0,  # Reduce repetition
        presence_penalty=0.0    # Encourage new topics
    )
    return response.choices[0].message.content

# Anthropic Claude
anthropic = Anthropic(api_key="your-key")

def call_claude(prompt):
    message = anthropic.messages.create(
        model="claude-3-opus-20240229",
        max_tokens=1000,
        temperature=0.7,
        messages=[{"role": "user", "content": prompt}]
    )
    return message.content

# Google Gemini
genai.configure(api_key="your-key")

def call_gemini(prompt):
    model = genai.GenerativeModel('gemini-pro')
    response = model.generate_content(prompt)
    return response.text

# Streaming responses for better UX
def stream_gpt4(prompt):
    stream = openai.ChatCompletion.create(
        model="gpt-4",
        messages=[{"role": "user", "content": prompt}],
        stream=True
    )
    
    for chunk in stream:
        if chunk.choices[0].delta.content:
            yield chunk.choices[0].delta.content