GGUF API Reference¶

class GGUFReader:
    def __init__(self, file_path: Union[str, Path])

from llamatelemetry.gguf_parser import GGUFReader

with GGUFReader("model.gguf") as reader:
    print(f"Model: {reader.metadata.get('general.name', 'unknown')}")
    print(f"Architecture: {reader.metadata.get('general.architecture')}")
    print(f"Tensors: {len(reader.tensors)}")

    for name, info in list(reader.tensors.items())[:5]:
        print(f"  {name}: {info.shape} ({info.ggml_type.name}, {info.nbytes / 1024**2:.1f} MB)")

    # Zero-copy tensor access
    data = reader.get_tensor_data("model.embed_tokens.weight")

from llamatelemetry.api.gguf import GGMLType

def gguf_report(
    model_path: str,
    include_raw_metadata: bool = False,
) -> Dict[str, Any]

from llamatelemetry.api.gguf import gguf_report
report = gguf_report("/models/gemma-3-1b-it-Q4_K_M.gguf")
print(f"Model: {report['name']}")
print(f"Size: {report['size_gb']} GB")
print(f"Architecture: {report['architecture']}")
print(f"Context: {report['context_length']}")

def report_model_suitability(
    model_path: str,
    ctx_size: int = 4096,
    dual_t4: bool = True,
) -> Dict[str, Any]

from llamatelemetry.api.gguf import report_model_suitability
report = report_model_suitability("/models/model.gguf", ctx_size=8192, dual_t4=True)
if report["fits"]:
    print(f"Model fits! Use {report['recommended_gpu_layers']} GPU layers")
else:
    print(f"Model too large. Try: {report['recommended_quant']}")

def quantization_matrix(as_dataframe: bool = False) -> Union[List[Dict], "pd.DataFrame"]

from llamatelemetry.api.gguf import quantization_matrix
for q in quantization_matrix():
    print(f"{q['name']:12s} {q['bits_per_weight']:.1f} bpw  quality={q['quality_score']}")

def estimate_gguf_size(param_count: int, quant_type: str) -> float

from llamatelemetry.api.gguf import estimate_gguf_size
size = estimate_gguf_size(7_000_000_000, "Q4_K_M")
print(f"Estimated: {size:.1f} GB")  # ~4.0 GB

def quantize(
    input_path: str,
    output_path: str,
    quant_type: str = "Q4_K_M",
    n_threads: Optional[int] = None,
    allow_requantize: bool = False,
    leave_output_tensor: bool = False,
    pure: bool = False,
    imatrix_path: Optional[str] = None,
    include_weights: Optional[List[str]] = None,
    exclude_weights: Optional[List[str]] = None,
    output_tensor_type: Optional[str] = None,
    token_embedding_type: Optional[str] = None,
    llama_quantize_path: Optional[str] = None,
) -> bool

def generate_imatrix(
    model_path: str,
    output_path: str,
    data_file: str,
    ctx_size: int = 512,
    n_batch: int = 512,
    n_threads: Optional[int] = None,
    n_gpu_layers: int = -1,
    llama_imatrix_path: Optional[str] = None,
) -> bool

from llamatelemetry.api.gguf import generate_imatrix, quantize

# Generate importance matrix
generate_imatrix("model-f16.gguf", "model.imatrix", "calibration.txt")

# Use it for I-quant quantization
quantize("model-f16.gguf", "model-iq4xs.gguf", "IQ4_XS", imatrix_path="model.imatrix")

def convert_hf_to_gguf(
    model_path: str,
    output_path: Optional[str] = None,
    outtype: str = "f16",
    vocab_only: bool = False,
    pad_vocab: bool = False,
    skip_unknown: bool = False,
    metadata: Optional[Dict[str, str]] = None,
    python_path: str = "python3",
    convert_script: Optional[str] = None,
) -> bool

def merge_lora(
    base_model: str,
    lora_path: str,
    output_path: str,
    lora_scale: float = 1.0,
    n_threads: Optional[int] = None,
    llama_export_path: Optional[str] = None,
) -> bool

def validate_gguf(path: str) -> Tuple[bool, str]

def get_model_summary(path: str) -> str

def compare_models(path1: str, path2: str) -> Dict[str, Any]

def find_gguf_models(directory: str, recursive: bool = True) -> List[str]

def recommend_quant_for_kaggle(
    param_count: int,
    dual_t4: bool = True,
    context_size: int = 4096,
    prefer_quality: bool = True,
) -> Dict[str, Any]

from llamatelemetry.api.gguf import recommend_quant_for_kaggle
rec = recommend_quant_for_kaggle(70_000_000_000, dual_t4=True)
print(f"Recommended: {rec['quant_type']} ({rec['estimated_size_gb']} GB)")
print(f"Fits: {rec['fits']}, Quality: {rec['quality_score']}/10")

def get_recommended_quant(original_size_gb: float, target_size_gb: float) -> str

@dataclass
class QuantTypeInfo:
    name: str              # e.g., "Q4_K_M"
    generation: str        # "legacy", "k-quant", or "i-quant"
    bits_per_weight: float # Approximate bits per weight
    quality_score: int     # 1-10 relative quality
    requires_imatrix: bool # Whether importance matrix recommended
    description: str       # Human-readable description

from llamatelemetry.gguf_parser import inspect_gguf
inspect_gguf("model.gguf")  # Prints detailed model info to stdout