Spaces:
Configuration error
Configuration error
| """ | |
| Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions` | |
| """ | |
| from typing import List, Optional, Tuple | |
| from litellm.secret_managers.main import get_secret_bool, get_secret_str | |
| from litellm.types.router import LiteLLM_Params | |
| from ...openai.chat.gpt_transformation import OpenAIGPTConfig | |
| class LiteLLMProxyChatConfig(OpenAIGPTConfig): | |
| def get_supported_openai_params(self, model: str) -> List: | |
| params_list = super().get_supported_openai_params(model) | |
| params_list.append("thinking") | |
| params_list.append("reasoning_effort") | |
| return params_list | |
| def _map_openai_params( | |
| self, | |
| non_default_params: dict, | |
| optional_params: dict, | |
| model: str, | |
| drop_params: bool, | |
| ) -> dict: | |
| supported_openai_params = self.get_supported_openai_params(model) | |
| for param, value in non_default_params.items(): | |
| if param == "thinking": | |
| optional_params.setdefault("extra_body", {})["thinking"] = value | |
| elif param in supported_openai_params: | |
| optional_params[param] = value | |
| return optional_params | |
| def _get_openai_compatible_provider_info( | |
| self, api_base: Optional[str], api_key: Optional[str] | |
| ) -> Tuple[Optional[str], Optional[str]]: | |
| api_base = api_base or get_secret_str("LITELLM_PROXY_API_BASE") # type: ignore | |
| dynamic_api_key = api_key or get_secret_str("LITELLM_PROXY_API_KEY") | |
| return api_base, dynamic_api_key | |
| def get_models( | |
| self, api_key: Optional[str] = None, api_base: Optional[str] = None | |
| ) -> List[str]: | |
| api_base, api_key = self._get_openai_compatible_provider_info(api_base, api_key) | |
| if api_base is None: | |
| raise ValueError( | |
| "api_base not set for LiteLLM Proxy route. Set in env via `LITELLM_PROXY_API_BASE`" | |
| ) | |
| models = super().get_models(api_key=api_key, api_base=api_base) | |
| return [f"litellm_proxy/{model}" for model in models] | |
| def get_api_key(api_key: Optional[str] = None) -> Optional[str]: | |
| return api_key or get_secret_str("LITELLM_PROXY_API_KEY") | |
| def _should_use_litellm_proxy_by_default( | |
| litellm_params: Optional[LiteLLM_Params] = None, | |
| ): | |
| """ | |
| Returns True if litellm proxy should be used by default for a given request | |
| Issue: https://github.com/BerriAI/litellm/issues/10559 | |
| Use case: | |
| - When using Google ADK, users want a flag to dynamically enable sending the request to litellm proxy or not | |
| - Allow the model name to be passed in original format and still use litellm proxy: | |
| "gemini/gemini-1.5-pro", "openai/gpt-4", "mistral/llama-2-70b-chat" etc. | |
| """ | |
| import litellm | |
| if get_secret_bool("USE_LITELLM_PROXY") is True: | |
| return True | |
| if litellm_params and litellm_params.use_litellm_proxy is True: | |
| return True | |
| if litellm.use_litellm_proxy is True: | |
| return True | |
| return False | |
| def litellm_proxy_get_custom_llm_provider_info( | |
| model: str, api_base: Optional[str] = None, api_key: Optional[str] = None | |
| ) -> Tuple[str, str, Optional[str], Optional[str]]: | |
| """ | |
| Force use litellm proxy for all models | |
| Issue: https://github.com/BerriAI/litellm/issues/10559 | |
| Expected behavior: | |
| - custom_llm_provider will be 'litellm_proxy' | |
| - api_base = api_base OR LITELLM_PROXY_API_BASE | |
| - api_key = api_key OR LITELLM_PROXY_API_KEY | |
| Use case: | |
| - When using Google ADK, users want a flag to dynamically enable sending the request to litellm proxy or not | |
| - Allow the model name to be passed in original format and still use litellm proxy: | |
| "gemini/gemini-1.5-pro", "openai/gpt-4", "mistral/llama-2-70b-chat" etc. | |
| Return model, custom_llm_provider, dynamic_api_key, api_base | |
| """ | |
| import litellm | |
| custom_llm_provider = "litellm_proxy" | |
| if model.startswith("litellm_proxy/"): | |
| model = model.split("/", 1)[1] | |
| ( | |
| api_base, | |
| api_key, | |
| ) = litellm.LiteLLMProxyChatConfig()._get_openai_compatible_provider_info( | |
| api_base=api_base, api_key=api_key | |
| ) | |
| return model, custom_llm_provider, api_key, api_base | |