Hua/nanobot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add vLLM/local LLM support

Browse Source 
- Add vllm provider configuration in config schema
- Auto-detect vLLM endpoints and use hosted_vllm/ prefix for LiteLLM
- Pass api_base directly to acompletion for custom endpoints
- Add vLLM status display in CLI status command
- Add vLLM setup documentation in README
This commit is contained in:
ZhihaoZhang97
2026-02-02 11:23:04 +11:00
parent 959c4dadf8
commit 2b19dcf9fd
4 changed files with 61 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
nanobot/cli/commands.py
View File

@@ -624,10 +624,13 @@ def status():
has_openrouter = bool(config.providers.openrouter.api_key)
has_anthropic = bool(config.providers.anthropic.api_key)
has_openai = bool(config.providers.openai.api_key)
has_vllm = bool(config.providers.vllm.api_base)
console.print(f"OpenRouter API: {'[green]✓[/green]' if has_openrouter else '[dim]not set[/dim]'}")
console.print(f"Anthropic API: {'[green]✓[/green]' if has_anthropic else '[dim]not set[/dim]'}")
console.print(f"OpenAI API: {'[green]✓[/green]' if has_openai else '[dim]not set[/dim]'}")
vllm_status = f"[green]✓ {config.providers.vllm.api_base}[/green]" if has_vllm else "[dim]not set[/dim]"
console.print(f"vLLM/Local: {vllm_status}")
if __name__ == "__main__":

8
nanobot/config/schema.py
View File

@@ -50,6 +50,7 @@ class ProvidersConfig(BaseModel):
anthropic: ProviderConfig = Field(default_factory=ProviderConfig)
openai: ProviderConfig = Field(default_factory=ProviderConfig)
openrouter: ProviderConfig = Field(default_factory=ProviderConfig)
vllm: ProviderConfig = Field(default_factory=ProviderConfig)
class GatewayConfig(BaseModel):
@@ -88,18 +89,21 @@ class Config(BaseSettings):
return Path(self.agents.defaults.workspace).expanduser()
def get_api_key(self) -> str | None:
"""Get API key in priority order: OpenRouter > Anthropic > OpenAI."""
"""Get API key in priority order: OpenRouter > Anthropic > OpenAI > vLLM."""
return (
self.providers.openrouter.api_key or
self.providers.anthropic.api_key or
self.providers.openai.api_key or
self.providers.vllm.api_key or
None
)
def get_api_base(self) -> str | None:
"""Get API base URL if using OpenRouter."""
"""Get API base URL if using OpenRouter or vLLM."""
if self.providers.openrouter.api_key:
return self.providers.openrouter.api_base or "https://openrouter.ai/api/v1"
if self.providers.vllm.api_base:
return self.providers.vllm.api_base
return None
class Config:

15
nanobot/providers/litellm_provider.py
View File

@@ -32,11 +32,17 @@ class LiteLLMProvider(LLMProvider):
(api_base and "openrouter" in api_base)
)
# Track if using custom endpoint (vLLM, etc.)
self.is_vllm = bool(api_base) and not self.is_openrouter
# Configure LiteLLM based on provider
if api_key:
if self.is_openrouter:
# OpenRouter mode - set key
os.environ["OPENROUTER_API_KEY"] = api_key
elif self.is_vllm:
# vLLM/custom endpoint - uses OpenAI-compatible API
os.environ["OPENAI_API_KEY"] = api_key
elif "anthropic" in default_model:
os.environ.setdefault("ANTHROPIC_API_KEY", api_key)
elif "openai" in default_model or "gpt" in default_model:
@@ -75,6 +81,11 @@ class LiteLLMProvider(LLMProvider):
if self.is_openrouter and not model.startswith("openrouter/"):
model = f"openrouter/{model}"
# For vLLM, use hosted_vllm/ prefix per LiteLLM docs
# Convert openai/ prefix to hosted_vllm/ if user specified it
if self.is_vllm:
model = f"hosted_vllm/{model}"
kwargs: dict[str, Any] = {
"model": model,
"messages": messages,
@@ -82,6 +93,10 @@ class LiteLLMProvider(LLMProvider):
"temperature": temperature,
}
# Pass api_base directly for custom endpoints (vLLM, etc.)
if self.api_base:
kwargs["api_base"] = self.api_base
if tools:
kwargs["tools"] = tools
kwargs["tool_choice"] = "auto"