feat(eval): add custom system prompt support for step-1 classification

This commit is contained in:
Roberto Musso
2026-04-06 22:56:30 +02:00
parent fe0dd038ee
commit 7e4374c69b
3 changed files with 126 additions and 36 deletions

View File

@@ -400,6 +400,7 @@ async def _classify_file(
projects: list[dict],
config_data_types: list[str],
langfuse_handler: Any | None = None,
custom_system_prompt: str | None = None,
) -> tuple[str, list[str], str | None]:
fallback: tuple[str, list[str], str | None] = ("new", list(config_data_types), None)
@@ -421,14 +422,20 @@ async def _classify_file(
if d in _DOMAIN_DESCRIPTIONS
)
system = tracing.compile_prompt(
"batch_file_classifier",
fallback=_STEP1_SYSTEM_PROMPT,
variables={
"domain_definitions": domain_definitions,
"projects_list": projects_list,
},
)
if custom_system_prompt:
# Fixture-provided prompt takes absolute priority
system = custom_system_prompt.format_map(
{"domain_definitions": domain_definitions, "projects_list": projects_list}
)
else:
system = tracing.compile_prompt(
"batch_file_classifier",
fallback=_STEP1_SYSTEM_PROMPT,
variables={
"domain_definitions": domain_definitions,
"projects_list": projects_list,
},
)
llm = get_llm(callbacks=[langfuse_handler] if langfuse_handler else None)
try: