diff --git a/nanobot/providers/custom_provider.py b/nanobot/providers/custom_provider.py index 56e6270..66df734 100644 --- a/nanobot/providers/custom_provider.py +++ b/nanobot/providers/custom_provider.py @@ -2,6 +2,7 @@ from __future__ import annotations +import uuid from typing import Any import json_repair @@ -15,7 +16,12 @@ class CustomProvider(LLMProvider): def __init__(self, api_key: str = "no-key", api_base: str = "http://localhost:8000/v1", default_model: str = "default"): super().__init__(api_key, api_base) self.default_model = default_model - self._client = AsyncOpenAI(api_key=api_key, base_url=api_base) + # Keep affinity stable for this provider instance to improve backend cache locality. + self._client = AsyncOpenAI( + api_key=api_key, + base_url=api_base, + default_headers={"x-session-affinity": uuid.uuid4().hex}, + ) async def chat(self, messages: list[dict[str, Any]], tools: list[dict[str, Any]] | None = None, model: str | None = None, max_tokens: int = 4096, temperature: float = 0.7,