diff --git a/python/ebook_search/llm_interface.py b/python/ebook_search/llm_interface.py index 6d39d7b..7fbe15c 100644 --- a/python/ebook_search/llm_interface.py +++ b/python/ebook_search/llm_interface.py @@ -174,6 +174,8 @@ async def request_chat_completion( client: httpx.AsyncClient, config: EbookSearchConfig, messages: Sequence[dict[str, str]], + *, + response_format: dict[str, object] | None = None, ) -> str: """Request a chat completion over a shared async client. @@ -181,6 +183,7 @@ async def request_chat_completion( client (httpx.AsyncClient): Shared async client whose connection pool bounds concurrency. config (EbookSearchConfig): Runtime settings supplying the endpoint, model, and auth. messages (Sequence[dict[str, str]]): OpenAI-style chat messages. + response_format (dict[str, object] | None): Optional OpenAI-compatible structured output constraint. Returns: str: The assistant message text. @@ -192,11 +195,8 @@ async def request_chat_completion( response = await client.post( f"{config.vllm_base_url.rstrip('/')}/chat/completions", headers=auth_headers(config.vllm_api_key), - json={ - "model": config.chat_model, - "messages": list(messages), - "temperature": 0, - }, + json={"model": config.chat_model, "messages": list(messages), "temperature": 0} + | ({"response_format": response_format} if response_format is not None else {}), timeout=config.chat_timeout_seconds, ) response.raise_for_status()