Spaces:
Configuration error
Configuration error
File size: 4,412 Bytes
447ebeb |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 |
"""
Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions`
"""
from typing import List, Optional, Tuple
from litellm.secret_managers.main import get_secret_bool, get_secret_str
from litellm.types.router import LiteLLM_Params
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
class LiteLLMProxyChatConfig(OpenAIGPTConfig):
def get_supported_openai_params(self, model: str) -> List:
params_list = super().get_supported_openai_params(model)
params_list.append("thinking")
params_list.append("reasoning_effort")
return params_list
def _map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
supported_openai_params = self.get_supported_openai_params(model)
for param, value in non_default_params.items():
if param == "thinking":
optional_params.setdefault("extra_body", {})["thinking"] = value
elif param in supported_openai_params:
optional_params[param] = value
return optional_params
def _get_openai_compatible_provider_info(
self, api_base: Optional[str], api_key: Optional[str]
) -> Tuple[Optional[str], Optional[str]]:
api_base = api_base or get_secret_str("LITELLM_PROXY_API_BASE") # type: ignore
dynamic_api_key = api_key or get_secret_str("LITELLM_PROXY_API_KEY")
return api_base, dynamic_api_key
def get_models(
self, api_key: Optional[str] = None, api_base: Optional[str] = None
) -> List[str]:
api_base, api_key = self._get_openai_compatible_provider_info(api_base, api_key)
if api_base is None:
raise ValueError(
"api_base not set for LiteLLM Proxy route. Set in env via `LITELLM_PROXY_API_BASE`"
)
models = super().get_models(api_key=api_key, api_base=api_base)
return [f"litellm_proxy/{model}" for model in models]
@staticmethod
def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
return api_key or get_secret_str("LITELLM_PROXY_API_KEY")
@staticmethod
def _should_use_litellm_proxy_by_default(
litellm_params: Optional[LiteLLM_Params] = None,
):
"""
Returns True if litellm proxy should be used by default for a given request
Issue: https://github.com/BerriAI/litellm/issues/10559
Use case:
- When using Google ADK, users want a flag to dynamically enable sending the request to litellm proxy or not
- Allow the model name to be passed in original format and still use litellm proxy:
"gemini/gemini-1.5-pro", "openai/gpt-4", "mistral/llama-2-70b-chat" etc.
"""
import litellm
if get_secret_bool("USE_LITELLM_PROXY") is True:
return True
if litellm_params and litellm_params.use_litellm_proxy is True:
return True
if litellm.use_litellm_proxy is True:
return True
return False
@staticmethod
def litellm_proxy_get_custom_llm_provider_info(
model: str, api_base: Optional[str] = None, api_key: Optional[str] = None
) -> Tuple[str, str, Optional[str], Optional[str]]:
"""
Force use litellm proxy for all models
Issue: https://github.com/BerriAI/litellm/issues/10559
Expected behavior:
- custom_llm_provider will be 'litellm_proxy'
- api_base = api_base OR LITELLM_PROXY_API_BASE
- api_key = api_key OR LITELLM_PROXY_API_KEY
Use case:
- When using Google ADK, users want a flag to dynamically enable sending the request to litellm proxy or not
- Allow the model name to be passed in original format and still use litellm proxy:
"gemini/gemini-1.5-pro", "openai/gpt-4", "mistral/llama-2-70b-chat" etc.
Return model, custom_llm_provider, dynamic_api_key, api_base
"""
import litellm
custom_llm_provider = "litellm_proxy"
if model.startswith("litellm_proxy/"):
model = model.split("/", 1)[1]
(
api_base,
api_key,
) = litellm.LiteLLMProxyChatConfig()._get_openai_compatible_provider_info(
api_base=api_base, api_key=api_key
)
return model, custom_llm_provider, api_key, api_base
|