Hua/nanobot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(custom-provider): keep instance-level session affinity header for cache locality

Browse Source
This commit is contained in:
Re-bin
2026-03-05 14:25:46 +00:00
parent a3c68ef140
commit fb77176cfd
1 changed files with 1 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
nanobot/providers/custom_provider.py
View File

@@ -16,6 +16,7 @@ class CustomProvider(LLMProvider):
def __init__(self, api_key: str = "no-key", api_base: str = "http://localhost:8000/v1", default_model: str = "default"): def __init__(self, api_key: str = "no-key", api_base: str = "http://localhost:8000/v1", default_model: str = "default"):
super().__init__(api_key, api_base) super().__init__(api_key, api_base)
self.default_model = default_model self.default_model = default_model
# Keep affinity stable for this provider instance to improve backend cache locality.
self._client = AsyncOpenAI( self._client = AsyncOpenAI(
api_key=api_key, api_key=api_key,
base_url=api_base, base_url=api_base,