feat(custom-provider): keep instance-level session affinity header for cache locality

This commit is contained in:
Re-bin
2026-03-05 14:25:46 +00:00
parent a3c68ef140
commit fb77176cfd

View File

@@ -16,6 +16,7 @@ class CustomProvider(LLMProvider):
def __init__(self, api_key: str = "no-key", api_base: str = "http://localhost:8000/v1", default_model: str = "default"):
super().__init__(api_key, api_base)
self.default_model = default_model
# Keep affinity stable for this provider instance to improve backend cache locality.
self._client = AsyncOpenAI(
api_key=api_key,
base_url=api_base,