Python SDK

from cli_sdk import (
    CLIClient, AsyncCLIClient, RetryConfig,                             # clients
    Belief, Set, Interval, Gate, Claim, Judge, Route,                   # queries
    BeliefAnswer, SetAnswer, IntervalAnswer, GateAnswer,                # answers
    ClaimAnswer, JudgeAnswer, RouteAnswer, EvaluateResponse, Guarantee, Usage,
    CalibrationProfile, CalibrationExample, LocalMonitor, Alert,        # calibration, monitoring
    OpenAIBackend, AzureOpenAIBackend, AnthropicBackend, GeminiBackend, # backends
    BedrockBackend, OpenRouterBackend, VLLMBackend, SGLangBackend, CustomBackend,
    CLIError, AuthenticationError, ValidationError,                     # errors
    InsufficientCalibrationError, RateLimitError, BackendError, ConfigurationError,
)

from cli_sdk import Belief, CLIClient, OpenAIBackend
 
with CLIClient(backend=OpenAIBackend(model="gpt-4.1-2025-04-14")) as client:   # default backend
    result = client.evaluate(
        context={"message": "Help! My payouts have been failing for 3 days."},
        queries={"urgent": Belief(instructions="Does this message convey urgency?",
                                  calibration_profile="urgency-v1")},
    )

result = client.evaluate(...)
result.answers          # dict: query id -> typed answer
result.usage            # Usage(backend_calls=..., backend_tokens=...)
result.request_id       # from the x-request-id header
result.warnings         # list[str]
result.backend          # dict
result.heuristic_answers  # ids of answers that carry no formal guarantee

from cli_sdk import CLIClient, EvaluateResponse, GateAnswer, Gate, OpenAIBackend, Set, SetAnswer
 
class RoutingResponse(EvaluateResponse):
    department: SetAnswer
    route: GateAnswer
 
with CLIClient() as client:
    result = client.evaluate(
        context={"ticket": "I was charged twice."},
        backend=OpenAIBackend(model="gpt-4.1-2025-04-14"),
        queries={
            "department": Set(
                instructions="Which team should handle this ticket?",
                options={"billing": "Payments, invoicing, refunds",
                         "technical": "Bugs, outages, integrations"},
                calibration_profile="support-routing-v3",
            ),
            "route": Gate(
                instructions="Auto-route this ticket without human review?",
                calibration_profile="support-routing-v3",
                guarantee="fdr",
                target=0.05,
            ),
        },
        response_model=RoutingResponse,
    )
 
assert result.department is result.answers["department"]

from cli_sdk import CLIClient, CustomBackend
 
class MyEngine(CustomBackend):
    access_level = "L3"
 
    def score_options(self, context, instructions, options):
        return my_model.option_probabilities(context, instructions, list(options))
 
    def sample(self, context, instructions, n):
        return [my_model.generate(context, instructions) for _ in range(n)]
 
    def score_text(self, context, instructions, candidate):
        return my_model.support_score(context, instructions, candidate)
 
client = CLIClient(backend=MyEngine(base_url="http://localhost:8000"))

client.calibration_profiles.create(name, backend, method="APS", alpha=0.10,
                                   group_by=None, prompt_template_hash=None,
                                   strict_fingerprint=True)        # -> CalibrationProfile
client.calibration_profiles.get(name)                              # -> CalibrationProfile
client.calibration_profiles.list()                                 # -> list[CalibrationProfile]
client.calibration_profiles.add_examples(name, examples)           # -> CalibrationProfile
client.calibration_profiles.label_with_judge(name, judge, unlabelled_examples,
                                             human_labelled_sample_size=300)  # -> dict
client.calibration_profiles.audit(name, fresh_examples)            # -> AuditResult
 
client.calibration_profiles.monitors.create(profile, type="coverage", target=None,
                                            false_alarm_rate=0.05,
                                            labelled_sample_rate=None,
                                            use_judge_pseudo_labels=False)   # -> Monitor
client.calibration_profiles.monitors.list(profile)                 # -> list[Monitor]
client.calibration_profiles.monitors.poll(profile, monitor_id=None)  # -> iterator of Alert

from cli_sdk import CLIClient, RetryConfig
 
client = CLIClient(retry=RetryConfig(max_attempts=5, base_delay=0.5, max_delay=8.0))

src/cli_sdk/
  client/          sync and async clients, retry policy
  queries/         Belief, Set, Interval, Gate, Claim, Judge, Route
  answers/         typed answers, the Guarantee card, EvaluateResponse
  calibration/     profiles, examples, audits, label-efficient calibration
  monitoring/      hosted monitors, alerts, LocalMonitor
  backends/        one adapter per provider, plus CustomBackend
  stats/           offline engine: conformal/, venn_abers/, evalues/
  cli_tool.py      the `cli` command
  exceptions.py    CLIError and subclasses
  constants.py     base URL, environment variable names, guarantee types, access levels