Hua/nanobot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1428: feat(custom-provider) session affinity header

Browse Source
This commit is contained in:
Re-bin
2026-03-05 14:27:24 +00:00
parent 7b676962ed fb77176cfd
commit bc52ad3dad
1 changed files with 7 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
nanobot/providers/custom_provider.py
View File

@@ -2,6 +2,7 @@
from __future__ import annotations from __future__ import annotations
import uuid
from typing import Any from typing import Any
import json_repair import json_repair
@@ -15,7 +16,12 @@ class CustomProvider(LLMProvider):
def __init__(self, api_key: str = "no-key", api_base: str = "http://localhost:8000/v1", default_model: str = "default"): def __init__(self, api_key: str = "no-key", api_base: str = "http://localhost:8000/v1", default_model: str = "default"):
super().__init__(api_key, api_base) super().__init__(api_key, api_base)
self.default_model = default_model self.default_model = default_model
self._client = AsyncOpenAI(api_key=api_key, base_url=api_base) # Keep affinity stable for this provider instance to improve backend cache locality.
self._client = AsyncOpenAI(
api_key=api_key,
base_url=api_base,
default_headers={"x-session-affinity": uuid.uuid4().hex},
)
async def chat(self, messages: list[dict[str, Any]], tools: list[dict[str, Any]] | None = None, async def chat(self, messages: list[dict[str, Any]], tools: list[dict[str, Any]] | None = None,
model: str | None = None, max_tokens: int = 4096, temperature: float = 0.7, model: str | None = None, max_tokens: int = 4096, temperature: float = 0.7,