Tutorial 01: Quickstart Guide¶

import subprocess
import os

print("="*70)
print("🔍 KAGGLE GPU ENVIRONMENT CHECK")
print("="*70)

# Check nvidia-smi
result = subprocess.run(["nvidia-smi", "-L"], capture_output=True, text=True)
gpu_lines = [l for l in result.stdout.strip().split("\n") if l.startswith("GPU")]
print(f"\n📊 Detected GPUs: {len(gpu_lines)}")
for line in gpu_lines:
    print(f"   {line}")

# Check CUDA version
print("\n📊 CUDA Version:")
!nvcc --version | grep release

# Check total VRAM
print("\n📊 VRAM Summary:")
!nvidia-smi --query-gpu=index,name,memory.total --format=csv

# Verify we have 2 GPUs
if len(gpu_lines) >= 2:
    print("\n✅ Multi-GPU environment confirmed! Ready for llamatelemetry v0.1.0.")
else:
    print("\n⚠️ WARNING: Less than 2 GPUs detected!")
    print("   Enable 'GPU T4 x2' in Kaggle notebook settings.")

%%time
# Install llamatelemetry v0.1.0 from GitHub
print("📦 Installing llamatelemetry v0.1.0...")
!pip install -q --no-cache-dir --force-reinstall git+https://github.com/llamatelemetry/llamatelemetry.git@v0.1.0

# Verify installation
import llamatelemetry
print(f"\n✅ llamatelemetry {llamatelemetry.__version__} installed!")

# Check llamatelemetry status
from llamatelemetry import check_cuda_available, get_cuda_device_info
from llamatelemetry.api.multigpu import gpu_count

cuda_info = get_cuda_device_info()
print(f"\n📊 llamatelemetry Status:")
print(f"   CUDA Available: {check_cuda_available()}")
print(f"   GPUs: {gpu_count()}")
if cuda_info:
    print(f"   CUDA Version: {cuda_info.get('cuda_version', 'N/A')}")

%%time
from huggingface_hub import hf_hub_download
import os

# Model selection - optimized for 15GB VRAM
MODEL_REPO = "unsloth/gemma-3-4b-it-GGUF"
MODEL_FILE = "gemma-3-4b-it-Q4_K_M.gguf"

print(f"📥 Downloading {MODEL_FILE}...")
print(f"   Repository: {MODEL_REPO}")
print(f"   Expected size: ~2.5GB")

# Download to Kaggle working directory
model_path = hf_hub_download(
    repo_id=MODEL_REPO,
    filename=MODEL_FILE,
    local_dir="/kaggle/working/models"
)

print(f"\n✅ Model downloaded: {model_path}")

# Show model size
size_gb = os.path.getsize(model_path) / (1024**3)
print(f"   Size: {size_gb:.2f} GB")

from llamatelemetry.server import ServerManager
from llamatelemetry.api.multigpu import kaggle_t4_dual_config

# Get optimized configuration for Kaggle T4×2
config = kaggle_t4_dual_config()

print("🚀 Starting llama-server with Multi-GPU configuration...")
print(f"   Model: {model_path}")
print(f"   GPU Layers: {config.n_gpu_layers} (all layers)")
print(f"   Context Size: {config.ctx_size}")
print(f"   Tensor Split: {config.tensor_split} (equal across 2 GPUs)")
print(f"   Flash Attention: {config.flash_attention}")

# Create server manager
server = ServerManager(server_url="http://127.0.0.1:8080")

# Start server with multi-GPU configuration
tensor_split_str = ",".join(str(x) for x in config.tensor_split)

server.start_server(
    model_path=model_path,
    host="127.0.0.1",
    port=8080,
    gpu_layers=config.n_gpu_layers,
    ctx_size=config.ctx_size,
    timeout=120,
    verbose=True,
    flash_attn=1 if config.flash_attention else 0,
    split_mode="layer",
    tensor_split=tensor_split_str,
)

print("\n✅ llama-server is ready with dual T4 GPUs!")
print(f"   API endpoint: http://127.0.0.1:8080")

from llamatelemetry.api.client import LlamaCppClient

# Create client
client = LlamaCppClient(base_url="http://127.0.0.1:8080")

# Test simple completion
print("💬 Testing inference...\n")

response = client.chat.create(
    messages=[
        {"role": "user", "content": "What is CUDA? Explain in 2 sentences."}
    ],
    max_tokens=100,
    temperature=0.7
)

print("📝 Response:")
print(response.choices[0].message.content)

print(f"\n📊 Stats:")
print(f"   Tokens generated: {response.usage.completion_tokens}")
print(f"   Total tokens: {response.usage.total_tokens}")

# Streaming example
print("💬 Streaming response...\n")

for chunk in client.chat.create(
    messages=[
        {"role": "user", "content": "Write a Python function to calculate factorial."}
    ],
    max_tokens=200,
    temperature=0.3,
    stream=True  # Enable streaming
):
    if hasattr(chunk, 'choices') and chunk.choices:
        delta = chunk.choices[0].delta
        if hasattr(delta, 'content') and delta.content:
            print(delta.content, end="", flush=True)

print("\n\n✅ Streaming complete!")

# Check GPU memory usage
print("📊 GPU Memory Usage:")
print("="*60)
!nvidia-smi --query-gpu=index,name,memory.used,memory.total,utilization.gpu --format=csv

print("\n💡 Note:")
print("   GPU 0: llama-server (LLM inference)")
print("   GPU 1: Available for RAPIDS/Graphistry")

# Stop the server
print("🛑 Stopping llama-server...")
server.stop_server()
print("\n✅ Server stopped. Resources freed.")

# Verify GPU memory is released
print("\n📊 GPU Memory After Cleanup:")
!nvidia-smi --query-gpu=index,memory.used,memory.total --format=csv