From f4eb5aebff8fdbff81c9c01d6fb7f892549b7796 Mon Sep 17 00:00:00 2001 From: aiapienthusiast <327629069+aiapienthusiast@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:45:43 +0400 Subject: [PATCH] Add CheaperInferenceLLM provider --- docs/modules/llm.md | 29 ++++++++++ docs/zh/modules/llm.md | 29 ++++++++++ evoagentx/models/__init__.py | 1 + evoagentx/models/cheaperinference_model.py | 61 ++++++++++++++++++++++ evoagentx/models/model_configs.py | 32 ++++++++++++ 5 files changed, 152 insertions(+) create mode 100644 evoagentx/models/cheaperinference_model.py diff --git a/docs/modules/llm.md b/docs/modules/llm.md index 0a4c0c49..9eda75bb 100644 --- a/docs/modules/llm.md +++ b/docs/modules/llm.md @@ -123,6 +123,35 @@ response = llm.generate( ) ``` +### CheaperInferenceLLM + +CheaperInferenceLLM is an adapter for [Cheaper Inference](https://cheaperinference.com/), an OpenAI-compatible LLM gateway. One API key gives access to models from several labs. + +The `CheaperInferenceLLM` model class in EvoAgentX uses the same API format for all models on Cheaper Inference. You can get an API key at [cheaperinference.com/signup](https://cheaperinference.com/signup). + +**Basic Usage:** + +```python +from evoagentx.models import CheaperInferenceConfig, CheaperInferenceLLM + +# Configure the model +config = CheaperInferenceConfig( + model="gpt-5.4-mini", + cheaperinference_key="your-cheaperinference-api-key", + temperature=0.7, + max_tokens=1000 +) + +# Initialize the model +llm = CheaperInferenceLLM(config=config) + +# Generate text +response = llm.generate( + prompt="Write a poem about artificial intelligence.", + system_message="You are a creative poet." +) +``` + ### OpenRouterLLM OpenRouterLLM is an adapter for the [OpenRouter platform](https://openrouter.ai/), which provides access to a wide range of language models from various providers through a unified API. It supports models from providers like Anthropic, Google, Meta, Mistral AI, and more, all accessible through a single interface. diff --git a/docs/zh/modules/llm.md b/docs/zh/modules/llm.md index be1adb2f..6b91b85a 100644 --- a/docs/zh/modules/llm.md +++ b/docs/zh/modules/llm.md @@ -122,6 +122,35 @@ response = llm.generate( ) ``` +### CheaperInferenceLLM + +CheaperInferenceLLM 是 [Cheaper Inference](https://cheaperinference.com/) 的适配器。Cheaper Inference 是一个兼容 OpenAI 的 LLM 网关，使用一个 API 密钥即可访问多家实验室的模型。 + +EvoAgentX 中的 `CheaperInferenceLLM` 模型类对 Cheaper Inference 上的所有模型使用相同的 API 格式。可以在 [cheaperinference.com/signup](https://cheaperinference.com/signup) 获取 API 密钥。 + +**基本用法：** + +```python +from evoagentx.models import CheaperInferenceConfig, CheaperInferenceLLM + +# Configure the model +config = CheaperInferenceConfig( + model="gpt-5.4-mini", + cheaperinference_key="your-cheaperinference-api-key", + temperature=0.7, + max_tokens=1000 +) + +# Initialize the model +llm = CheaperInferenceLLM(config=config) + +# Generate text +response = llm.generate( + prompt="Write a poem about artificial intelligence.", + system_message="You are a creative poet." +) +``` + ### OpenRouterLLM OpenRouterLLM 是 [OpenRouter 平台](https://openrouter.ai/) 的适配器，该平台通过统一的 API 提供对各种提供商的语言模型的访问。它支持来自 Anthropic、Google、Meta、Mistral AI 等提供商的模型，所有这些都可以通过单一接口访问。 diff --git a/evoagentx/models/__init__.py b/evoagentx/models/__init__.py index 1e37a69f..77a326de 100644 --- a/evoagentx/models/__init__.py +++ b/evoagentx/models/__init__.py @@ -8,3 +8,4 @@ from .openrouter_model import * from .aliyun_model import * from .novita_model import * +from .cheaperinference_model import * diff --git a/evoagentx/models/cheaperinference_model.py b/evoagentx/models/cheaperinference_model.py new file mode 100644 index 00000000..baf7220f --- /dev/null +++ b/evoagentx/models/cheaperinference_model.py @@ -0,0 +1,61 @@ +from typing import Union + +from openai import AsyncOpenAI, OpenAI +from openai.types.chat import ChatCompletion, ChatCompletionChunk + +from .openai_model import OpenAILLM +from .model_configs import CheaperInferenceConfig +from ..core.logging import logger +from ..core.registry import register_model +from .model_utils import Cost, cost_manager + +# Cheaper Inference exposes an OpenAI-compatible API at this base URL. +CHEAPERINFERENCE_BASE_URL = "https://api.cheaperinference.com/v1" + + +@register_model(config_cls=CheaperInferenceConfig, alias=["cheaperinference"]) +class CheaperInferenceLLM(OpenAILLM): + """Cheaper Inference LLM client. + + Cheaper Inference speaks the OpenAI chat-completions protocol, so this reuses all of + ``OpenAILLM``'s generation/streaming/tool-call logic and only overrides client + construction and cost handling. + + LiteLLM has no pricing data for Cheaper Inference models, so the dollar cost + cannot be computed or approximated. Token counts are still tracked; the + per-model cost is recorded as 0. A warning is emitted once at init time. + """ + + def init_model(self): + self._client = None + self._async_client = None + # parameters in CheaperInferenceConfig that are not model input parameters + self._default_ignore_fields = ["llm_type", "cheaperinference_key", "output_response"] + logger.warning( + "[CheaperInferenceLLM] LiteLLM has no pricing data for Cheaper Inference models, so dollar " + "cost cannot be computed. Token usage will be tracked, but cost will " + "be recorded as 0." + ) + + def _init_client(self, config: CheaperInferenceConfig): + return OpenAI(api_key=config.cheaperinference_key, base_url=CHEAPERINFERENCE_BASE_URL) + + def _init_async_client(self, config: CheaperInferenceConfig): + return AsyncOpenAI(api_key=config.cheaperinference_key, base_url=CHEAPERINFERENCE_BASE_URL) + + def _update_cost(self, response: Union[ChatCompletion, ChatCompletionChunk]): + # Override OpenAILLM's LiteLLM-based cost computation: only record token + # counts and leave cost at 0 (see class docstring). + usage = getattr(response, "usage", None) + if usage is None: + logger.warning( + f"[CheaperInferenceLLM] usage is None in response (id={getattr(response, 'id', '?')}); " + "tokens will not be recorded." + ) + return + cost = Cost( + input_tokens=usage.prompt_tokens, + output_tokens=usage.completion_tokens, + cost=0.0, + ) + cost_manager.update_cost(cost=cost, model=self.config.model) diff --git a/evoagentx/models/model_configs.py b/evoagentx/models/model_configs.py index ed8a545c..bb72614b 100644 --- a/evoagentx/models/model_configs.py +++ b/evoagentx/models/model_configs.py @@ -172,6 +172,38 @@ def __str__(self): return self.model +class CheaperInferenceConfig(LLMConfig): + + # LLM keys + llm_type: str = "CheaperInferenceLLM" + cheaperinference_key: Optional[str] = Field(default=None, description="the API key used to authenticate Cheaper Inference requests") + + # generation parameters + temperature: Optional[float] = Field(default=None, description="the temperature used to scaling logits") + max_tokens : Optional[int] = Field(default=None, description="maximum number of generated tokens") + max_completion_tokens: Optional[int] = Field(default=None, description="An upper bound for the number of tokens that can be generated for a completion, including visible output tokens and reasoning tokens. Commonly used in OpenAI's o1 series models.") + top_p: Optional[float] = Field(default=None, description="Only sample from tokens with cumulative probability greater than top_p when generating text.") + n: Optional[int] = Field(default=None, description="How many chat completion choices to generate for each input message.") + stream: Optional[bool] = Field(default=None, description=" If set to true, it sends partial message deltas. Tokens will be sent as they become available, with the stream terminated by a [DONE] message.") + stream_options: Optional[dict] = Field(default=None, description="Options for streaming response. Only set this when you set stream: true") + timeout: Optional[Union[float, int]] = Field(default=None, description="Timeout in seconds for completion requests (Defaults to 600 seconds)") + + # tools + tools: Optional[List] = Field(default=None, description="A list of tools the model may call. Currently, only functions are supported as a tool. Use this to provide a list of functions the model may generate JSON inputs for.") + tool_choice: Optional[Union[str, dict]] = Field(default=None, description="Controls which (if any) function is called by the model. none means the model will not call a function and instead generates a message. auto means the model can pick between generating a message or calling a function. required forces the model to call a tool. Specifying a particular function via {\"type\": \"function\", \"function\": {\"name\": \"my_function\"}} forces the model to call that function.") + parallel_tool_calls: Optional[bool] = Field(default=None, description="Whether to enable parallel function calling during tool use. OpenAI default is true.") + + # token probabilities + logprobs: Optional[bool] = Field(default=None, description="Whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each output token returned in the content of message.") + top_logprobs: Optional[int] = Field(default=None, description="An integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with an associated log probability. logprobs must be set to true if this parameter is used.") + + # output format + response_format: Optional[Union[BaseModel, dict]] = Field(default=None, description=" An object specifying the format that the model must output.") + + def __str__(self): + return self.model + + # def get_default_device(): # return "cuda" if torch.cuda.is_available() else "cpu"