971f1dd84f
- tracing.py: init/shutdown, trace_span, get_langfuse_callback, prompt mgmt - main.py: init_langfuse at startup, shutdown on teardown - redis_consumer.py: trace_span around journey_start/message/agent_trigger - agent_runner.py: thread langfuse_handler through classify + processing LLM - journey.py: thread langfuse_handler through _call_llm_with_tools - llm.py: accept callbacks param, forward to LLM constructors - requirements.txt: add langfuse>=3.0.0
79 lines
2.1 KiB
Python
79 lines
2.1 KiB
Python
"""LLM factory — centralised model instantiation via LiteLLM.
|
|
|
|
Identical to services/chat/app/llm.py. Uses shared.config.settings.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import warnings
|
|
|
|
from openai import AsyncOpenAI
|
|
import litellm
|
|
|
|
from langchain_openai import ChatOpenAI
|
|
from langchain_litellm import ChatLiteLLM
|
|
|
|
from shared.config import settings
|
|
|
|
litellm.drop_params = True
|
|
|
|
warnings.filterwarnings(
|
|
"ignore",
|
|
message=r"PydanticSerializationUnexpectedValue\(Expected `ResponseAPIUsage`",
|
|
category=UserWarning,
|
|
)
|
|
|
|
|
|
def _api_key_for_model(model: str) -> str | None:
|
|
if model.startswith("anthropic/"):
|
|
return settings.ANTHROPIC_API_KEY or None
|
|
if model.startswith("gemini/") or model.startswith("google/"):
|
|
return settings.GOOGLE_API_KEY or None
|
|
if model.startswith("cerebras/"):
|
|
return settings.CEREBRAS_API_KEY or None
|
|
if model.startswith("github_copilot/"):
|
|
return None
|
|
return settings.OPENAI_API_KEY or None
|
|
|
|
|
|
def get_llm(
|
|
*,
|
|
model: str | None = None,
|
|
temperature: float = 0,
|
|
callbacks: list | None = None,
|
|
) -> ChatOpenAI | ChatLiteLLM:
|
|
model = model or settings.LLM_MODEL
|
|
|
|
if settings.GITHUB_COPILOT_TOKEN_DIR:
|
|
os.environ.setdefault("GITHUB_COPILOT_TOKEN_DIR", settings.GITHUB_COPILOT_TOKEN_DIR)
|
|
|
|
if "/" in model:
|
|
return ChatLiteLLM(model=model, temperature=temperature, callbacks=callbacks)
|
|
|
|
return ChatOpenAI(
|
|
model=model,
|
|
temperature=temperature,
|
|
api_key=_api_key_for_model(model),
|
|
callbacks=callbacks,
|
|
)
|
|
|
|
|
|
def get_router_llm(
|
|
*,
|
|
temperature: float = 0,
|
|
) -> ChatOpenAI | ChatLiteLLM:
|
|
return get_llm(model=settings.LLM_ROUTER_MODEL, temperature=temperature)
|
|
|
|
|
|
async def embed(text: str) -> list[float]:
|
|
model = settings.LLM_EMBED_MODEL
|
|
|
|
if model.startswith("github_copilot/") or "/" in model:
|
|
response = await litellm.aembedding(model=model, input=[text])
|
|
return response.data[0]["embedding"]
|
|
|
|
client = AsyncOpenAI(api_key=settings.OPENAI_API_KEY)
|
|
response = await client.embeddings.create(model=model, input=text)
|
|
return response.data[0].embedding
|