From ecdfaf0a5a00b0772719aa306d4cb36d8512f9c7 Mon Sep 17 00:00:00 2001 From: David Markey Date: Sun, 1 Mar 2026 20:49:00 +0000 Subject: [PATCH] feat(custom-provider): add x-session-affinity header for prompt caching --- nanobot/providers/custom_provider.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/nanobot/providers/custom_provider.py b/nanobot/providers/custom_provider.py index 56e6270..02183f3 100644 --- a/nanobot/providers/custom_provider.py +++ b/nanobot/providers/custom_provider.py @@ -2,6 +2,7 @@ from __future__ import annotations +import uuid from typing import Any import json_repair @@ -15,7 +16,11 @@ class CustomProvider(LLMProvider): def __init__(self, api_key: str = "no-key", api_base: str = "http://localhost:8000/v1", default_model: str = "default"): super().__init__(api_key, api_base) self.default_model = default_model - self._client = AsyncOpenAI(api_key=api_key, base_url=api_base) + self._client = AsyncOpenAI( + api_key=api_key, + base_url=api_base, + default_headers={"x-session-affinity": uuid.uuid4().hex}, + ) async def chat(self, messages: list[dict[str, Any]], tools: list[dict[str, Any]] | None = None, model: str | None = None, max_tokens: int = 4096, temperature: float = 0.7,