feat(batch-agent): integrate Langfuse tracing

- tracing.py: init/shutdown, trace_span, get_langfuse_callback, prompt mgmt
- main.py: init_langfuse at startup, shutdown on teardown
- redis_consumer.py: trace_span around journey_start/message/agent_trigger
- agent_runner.py: thread langfuse_handler through classify + processing LLM
- journey.py: thread langfuse_handler through _call_llm_with_tools
- llm.py: accept callbacks param, forward to LLM constructors
- requirements.txt: add langfuse>=3.0.0

services/batch-agent/app/agent_runner.py

@@ -193,9 +193,11 @@ async def _run_agent_with_tools(
     user_message: str,
     tools: list[Any],
     max_steps: int,
+    langfuse_handler: Any | None = None,
 ) -> str:
     """Run an LLM agent with tool-calling, returning the final text response."""
-    llm = get_llm()
+    callbacks = [langfuse_handler] if langfuse_handler else None
+    llm = get_llm(callbacks=callbacks)
     llm_with_tools = llm.bind_tools(tools)
     messages: list[Any] = [
         SystemMessage(content=system_prompt),
@@ -396,6 +398,7 @@ async def _classify_file(
     file_content: str,
     projects: list[dict],
     config_data_types: list[str],
+    langfuse_handler: Any | None = None,
 ) -> tuple[str, list[str], str | None]:
     fallback: tuple[str, list[str], str | None] = ("new", list(config_data_types), None)
 
@@ -422,7 +425,7 @@ async def _classify_file(
         projects_list=projects_list,
     )
 
-    llm = get_llm()
+    llm = get_llm(callbacks=[langfuse_handler] if langfuse_handler else None)
     try:
         response = await llm.ainvoke([
             SystemMessage(content=system),
@@ -458,7 +461,7 @@ async def _classify_file(
 # ── Local agent runner (two-step per file) ────────────────────────────────
 
 
-async def run_local_agent(user_id: str, trigger_data: dict[str, Any]) -> None:
+async def run_local_agent(user_id: str, trigger_data: dict[str, Any], *, langfuse_handler: Any | None = None) -> None:
     """Execute a local directory agent run.
 
     In the microservice world, trigger_data is a serialized dict from
@@ -552,6 +555,7 @@ async def run_local_agent(user_id: str, trigger_data: dict[str, Any]) -> None:
                     file_content=file_content,
                     projects=projects,
                     config_data_types=data_types,
+                    langfuse_handler=langfuse_handler,
                 )
 
                 # Step 2 — resolve project_id, fetch entities, process
@@ -610,6 +614,7 @@ async def run_local_agent(user_id: str, trigger_data: dict[str, Any]) -> None:
                     ),
                     tools=processing_tools,
                     max_steps=_MAX_PROCESSING_STEPS,
+                    langfuse_handler=langfuse_handler,
                 )
                 logger.info(
                     "agent_runner: run=%s file=%r result=%s",
@@ -660,7 +665,7 @@ async def run_local_agent(user_id: str, trigger_data: dict[str, Any]) -> None:
 _CLOUD_DEFAULT_LOOKBACK_DAYS: int = 7
 
 
-async def run_cloud_agent(user_id: str, config_id: str) -> None:
+async def run_cloud_agent(user_id: str, config_id: str, *, langfuse_handler: Any | None = None) -> None:
     """Execute a cloud connector agent run.
 
     Loads the CloudAgentConfig from DB, decrypts OAuth tokens, fetches
@@ -789,6 +794,7 @@ async def run_cloud_agent(user_id: str, config_id: str) -> None:
                     user_message=f"Process this message content:\n\n{content_text[:8000]}",
                     tools=processing_tools,
                     max_steps=_MAX_PROCESSING_STEPS,
+                    langfuse_handler=langfuse_handler,
                 )
             except Exception as exc:
                 errors.append(f"LLM processing error for message {msg.id!r}: {exc}")