Client API Reference¶

class LlamaCppClient:
    def __init__(
        self,
        base_url: str = "http://127.0.0.1:8080",
        api_key: Optional[str] = None,
        timeout: float = 600.0,
        verify_ssl: bool = True,
    )

from llamatelemetry.api.client import LlamaCppClient

client = LlamaCppClient("http://localhost:8080")

def health(self) -> HealthStatus

def is_ready(self) -> bool

def wait_until_ready(self, timeout: float = 60.0, poll_interval: float = 1.0) -> bool

def props(self) -> Dict[str, Any]

def set_props(self, **kwargs) -> Dict[str, Any]

def metrics(self) -> str

def chat_completion(
    self,
    messages: List[Dict[str, Any]],
    **kwargs,
) -> Union[CompletionResponse, Iterator[Dict[str, Any]]]

response = client.chat_completion(
    messages=[
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": "What is CUDA?"},
    ],
    max_tokens=200,
    temperature=0.7,
)
print(response.choices[0].message.content)

def create(
    self,
    messages: List[Dict[str, Any]],
    model: str = "gpt-codex-5.3",
    max_tokens: Optional[int] = None,
    temperature: float = 1.0,
    top_p: float = 1.0,
    n: int = 1,
    stream: bool = False,
    stop: Optional[Union[str, List[str]]] = None,
    presence_penalty: float = 0.0,
    frequency_penalty: float = 0.0,
    logit_bias: Optional[Dict[str, float]] = None,
    user: Optional[str] = None,
    response_format: Optional[Dict[str, Any]] = None,
    tools: Optional[List[Dict[str, Any]]] = None,
    tool_choice: Optional[Union[str, Dict[str, Any]]] = None,
    seed: Optional[int] = None,
    # llama.cpp-specific extensions
    mirostat: int = 0,
    mirostat_tau: float = 5.0,
    mirostat_eta: float = 0.1,
    grammar: Optional[str] = None,
    min_p: float = 0.05,
    top_k: int = 40,
    repeat_penalty: float = 1.1,
    **kwargs,
) -> Union[CompletionResponse, Iterator[Dict[str, Any]]]

# Standard chat
response = client.chat.completions.create(
    messages=[{"role": "user", "content": "Hello!"}],
    max_tokens=100,
    temperature=0.7,
)
print(response.choices[0].message.content)

# Streaming
for chunk in client.chat.completions.create(
    messages=[{"role": "user", "content": "Tell me a story"}],
    stream=True,
    max_tokens=500,
):
    print(chunk.get("choices", [{}])[0].get("delta", {}).get("content", ""), end="")

# Structured output with JSON schema
response = client.chat.completions.create(
    messages=[{"role": "user", "content": "List 3 colors"}],
    response_format={"type": "json_schema", "json_schema": {
        "name": "colors", "schema": {"type": "object", "properties": {
            "colors": {"type": "array", "items": {"type": "string"}}
        }}
    }},
)

def complete(
    self,
    prompt: Union[str, List[Union[str, int]]],
    n_predict: int = -1,
    temperature: float = 0.8,
    top_k: int = 40,
    top_p: float = 0.95,
    min_p: float = 0.05,
    repeat_penalty: float = 1.1,
    repeat_last_n: int = 64,
    presence_penalty: float = 0.0,
    frequency_penalty: float = 0.0,
    mirostat: int = 0,
    mirostat_tau: float = 5.0,
    mirostat_eta: float = 0.1,
    grammar: Optional[str] = None,
    json_schema: Optional[Dict[str, Any]] = None,
    seed: int = -1,
    stop: Optional[List[str]] = None,
    stream: bool = False,
    cache_prompt: bool = True,
    n_probs: int = 0,
    samplers: Optional[List[str]] = None,
    dry_multiplier: float = 0.0,
    dry_base: float = 1.75,
    dry_allowed_length: int = 2,
    dry_penalty_last_n: int = -1,
    xtc_probability: float = 0.0,
    xtc_threshold: float = 0.1,
    dynatemp_range: float = 0.0,
    dynatemp_exponent: float = 1.0,
    typical_p: float = 1.0,
    id_slot: int = -1,
    return_tokens: bool = False,
    **kwargs,
) -> Union[CompletionResponse, Iterator[Dict[str, Any]]]

response = client.complete(
    prompt="The capital of France is",
    n_predict=50,
    temperature=0.7,
)
print(response.choices[0].text)

def simple_completion(self, prompt: str, **kwargs) -> Union[str, CompletionResponse, Iterator]

def create(
    self,
    input: Union[str, List[str]],
    model: str = "text-embedding-ada-002",
    encoding_format: str = "float",
    dimensions: Optional[int] = None,
) -> EmbeddingsResponse

def embed(
    self,
    content: Union[str, List[str]],
    embd_normalize: int = 2,
) -> List[List[float]]

def tokenize(
    self,
    content: str,
    add_special: bool = False,
    parse_special: bool = True,
    with_pieces: bool = False,
) -> TokenizeResponse

def detokenize(self, tokens: List[int]) -> str

tokens = client.tokenize("Hello, world!")
print(tokens.tokens)  # [15496, 11, 995, 0]
text = client.detokenize(tokens.tokens)
print(text)  # "Hello, world!"

def rerank(
    self,
    query: str,
    documents: List[str],
    top_n: Optional[int] = None,
) -> RerankResponse

results = client.rerank(
    query="What is a panda?",
    documents=["A panda is a bear", "Hello world", "Pandas eat bamboo"],
    top_n=2,
)
for r in results.results:
    print(f"  [{r.index}] score={r.relevance_score:.3f}")

def infill(
    self,
    input_prefix: str,
    input_suffix: str,
    input_extra: Optional[List[Dict[str, str]]] = None,
    prompt: Optional[str] = None,
    stream: bool = False,
    **kwargs,
) -> Union[CompletionResponse, Iterator[Dict[str, Any]]]

def list(self, fail_on_no_slot: bool = False) -> List[SlotInfo]

def save(self, slot_id: int, filename: str) -> Dict[str, Any]
def restore(self, slot_id: int, filename: str) -> Dict[str, Any]
def erase(self, slot_id: int) -> Dict[str, Any]

def list(self) -> List[LoraAdapter]

def set_scales(self, adapters: List[Dict[str, Any]]) -> bool

client.lora.set_scales([
    {"id": 0, "scale": 0.5},
    {"id": 1, "scale": 0.8},
])