fix(tracing): use Langfuse compile_prompt with {{variable}} syntax

- tracing.py: add compile_prompt() that uses Langfuse .compile(**vars)
  for {{variable}} substitution, falls back to Python .format() for
  hardcoded {variable} templates
- agent_runner.py: replace _get_system_prompt().format() with
  tracing.compile_prompt() for batch_file_classifier, batch_processing,
  batch_cloud_processing prompts
- journey.py: replace get_prompt + .format() with compile_prompt()
  for journey_system prompt
- chat tracing.py: add compile_prompt() for parity (chat prompts
  currently have no variables, but ready for future use)
- Remove unused _get_system_prompt helper

services/batch-agent/app/tracing.py

@@ -167,9 +167,9 @@ def get_prompt(
     fallback: str | None = None,
     cache_ttl_seconds: int = 300,
 ) -> str | None:
-    """Fetch a managed prompt from Langfuse by name.
+    """Fetch a managed prompt from Langfuse by name (without variable compilation).
 
-    Returns the compiled prompt string, or *fallback* if the prompt is not
+    Returns the raw prompt string, or *fallback* if the prompt is not
     found or Langfuse is disabled.
     """
     lf = _get_client()
@@ -192,6 +192,46 @@ def get_prompt(
         return fallback
 
 
+def compile_prompt(
+    name: str,
+    *,
+    fallback: str,
+    variables: dict[str, str],
+    version: int | None = None,
+    label: str | None = None,
+    cache_ttl_seconds: int = 300,
+) -> str:
+    """Fetch a managed prompt from Langfuse and compile it with ``{{variables}}``.
+
+    If the prompt exists in Langfuse, uses the SDK's ``.compile(**variables)``
+    which replaces ``{{key}}`` placeholders.  If Langfuse is disabled or the
+    prompt is not found, falls back to ``fallback.format(**variables)`` (Python
+    ``{key}`` placeholders).
+
+    This means:
+      - Langfuse prompts use ``{{variable}}`` syntax.
+      - Hardcoded fallback strings use Python ``{variable}`` syntax.
+    """
+    lf = _get_client()
+    if lf is None:
+        return fallback.format(**variables)
+
+    try:
+        kwargs: dict[str, Any] = {
+            "name": name,
+            "cache_ttl_seconds": cache_ttl_seconds,
+        }
+        if version is not None:
+            kwargs["version"] = version
+        if label is not None:
+            kwargs["label"] = label
+        prompt = lf.get_prompt(**kwargs)
+        return prompt.compile(**variables)
+    except Exception as exc:
+        logger.warning("tracing: compile_prompt(%s) failed, using fallback: %s", name, exc)
+        return fallback.format(**variables)
+
+
 def link_prompt_to_trace(
     span: Any,
     prompt_name: str,