adiuvAI/api
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(batch-agent): add journey eval to E2E harness

Browse Source 
- journey_runner.py: orchestrates journey start → simulated user
  messages → template extraction → LLM judge scoring
- config.py: JourneyFixture dataclass with user_messages and
  expected_template_criteria, discover_journey_fixtures()
- langfuse_eval.py: sync_journey_fixture_to_dataset()
- cli.py: new 'journey' subcommand (python -m eval journey)
  with --fixture, --models, --judge-model flags
- fixtures/journey_invoice_setup.yaml: example journey fixture
  with 4 user messages and 8 quality criteria
This commit is contained in:
Roberto Musso
2026-03-23 23:16:41 +01:00
parent d856dfd28c
commit 63fa119543
5 changed files with 643 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

101
services/batch-agent/eval/cli.py
View File

@@ -3,13 +3,17 @@
Usage::
# From services/batch-agent/:
python -m eval run # all fixtures, default model
python -m eval run # all agent fixtures, default model
python -m eval run --fixture=freelance-invoices # single fixture
python -m eval run --models=gpt-4o,anthropic/claude-sonnet-4
python -m eval run --variants=baseline,detailed # specific prompt variants
python -m eval run --no-judge # skip LLM judge scoring
python -m eval list # list available fixtures
python -m eval journey # all journey fixtures
python -m eval journey --fixture=journey-invoices # single journey fixture
python -m eval journey --models=gpt-4o,anthropic/claude-sonnet-4
python -m eval list # list all fixtures
python -m eval sync # sync fixtures to Langfuse datasets
"""
@@ -28,8 +32,9 @@ for p in (_SERVICE_ROOT, _REPO_ROOT):
if str(p) not in sys.path:
sys.path.insert(0, str(p))
from eval.config import discover_fixtures
from eval.config import discover_fixtures, discover_journey_fixtures
from eval.runner import run_fixture_eval, print_results
from eval.journey_runner import run_journey_fixture_eval, print_journey_results
from eval import langfuse_eval
@@ -90,6 +95,29 @@ def _parse_args() -> argparse.Namespace:
list_cmd.add_argument("--fixtures-dir", default=None)
list_cmd.add_argument("-v", "--verbose", action="store_true")
# ── journey ───────────────────────────────────────────────────
journey_cmd = sub.add_parser("journey", help="Run journey evaluations")
journey_cmd.add_argument(
"--fixture", "-f",
help="Run only the named journey fixture (default: all)",
)
journey_cmd.add_argument(
"--models", "-m",
default="gpt-4o",
help="Comma-separated list of models to test (default: gpt-4o)",
)
journey_cmd.add_argument(
"--judge-model",
default="gpt-4o-mini",
help="Model for LLM judge (default: gpt-4o-mini)",
)
journey_cmd.add_argument(
"--fixtures-dir",
default=None,
help="Path to fixtures directory (default: eval/fixtures/)",
)
journey_cmd.add_argument("-v", "--verbose", action="store_true")
# ── sync ──────────────────────────────────────────────────────
sync_cmd = sub.add_parser("sync", help="Sync fixtures to Langfuse datasets")
sync_cmd.add_argument("--fixture", "-f", default=None, help="Sync only the named fixture")
@@ -136,25 +164,41 @@ async def _cmd_run(args: argparse.Namespace) -> None:
def _cmd_list(args: argparse.Namespace) -> None:
fixtures = discover_fixtures(_fixtures_dir(args.fixtures_dir))
if not fixtures:
journey_fixtures = discover_journey_fixtures(_fixtures_dir(args.fixtures_dir))
if not fixtures and not journey_fixtures:
print("No fixtures found.")
return
print(f"\n{'Name':<30} {'Types':<25} {'Variants':<20} {'Expected'}")
print("-" * 90)
for f in fixtures:
variants = ", ".join(f.prompt_variants.keys())
types = ", ".join(f.data_types)
print(f"{f.name:<30} {types:<25} {variants:<20} {len(f.expected)}")
if fixtures:
print(f"\n{'[Agent Fixtures]'}")
print(f"{'Name':<30} {'Types':<25} {'Variants':<20} {'Expected'}")
print("-" * 90)
for f in fixtures:
variants = ", ".join(f.prompt_variants.keys())
types = ", ".join(f.data_types)
print(f"{f.name:<30} {types:<25} {variants:<20} {len(f.expected)}")
if journey_fixtures:
print(f"\n{'[Journey Fixtures]'}")
print(f"{'Name':<30} {'Types':<25} {'Messages':<10} {'Criteria'}")
print("-" * 90)
for f in journey_fixtures:
types = ", ".join(f.data_types)
print(f"{f.name:<30} {types:<25} {len(f.user_messages):<10} {len(f.expected_template_criteria)}")
print()
def _cmd_sync(args: argparse.Namespace) -> None:
fixtures = discover_fixtures(_fixtures_dir(args.fixtures_dir))
journey_fixtures = discover_journey_fixtures(_fixtures_dir(args.fixtures_dir))
if args.fixture:
fixtures = [f for f in fixtures if f.name == args.fixture]
journey_fixtures = [f for f in journey_fixtures if f.name == args.fixture]
if not fixtures:
if not fixtures and not journey_fixtures:
print("No fixtures to sync.")
return
@@ -165,6 +209,39 @@ def _cmd_sync(args: argparse.Namespace) -> None:
else:
print(f"Skipped: {fixture.name} (Langfuse not configured)")
for fixture in journey_fixtures:
name = langfuse_eval.sync_journey_fixture_to_dataset(fixture)
if name:
print(f"Synced: {fixture.name} → {name}")
else:
print(f"Skipped: {fixture.name} (Langfuse not configured)")
async def _cmd_journey(args: argparse.Namespace) -> None:
journey_fixtures = discover_journey_fixtures(_fixtures_dir(args.fixtures_dir))
if not journey_fixtures:
print("No journey fixtures found. Create YAML files with type: journey in eval/fixtures/.")
return
if args.fixture:
journey_fixtures = [f for f in journey_fixtures if f.name == args.fixture]
if not journey_fixtures:
print(f"Journey fixture '{args.fixture}' not found.")
return
models = [m.strip() for m in args.models.split(",")]
all_results = []
for fixture in journey_fixtures:
results = await run_journey_fixture_eval(
fixture,
models=models,
judge_model=args.judge_model,
)
all_results.extend(results)
print_journey_results(all_results)
def main() -> None:
args = _parse_args()
@@ -172,6 +249,8 @@ def main() -> None:
if args.command == "run":
asyncio.run(_cmd_run(args))
elif args.command == "journey":
asyncio.run(_cmd_journey(args))
elif args.command == "list":
_cmd_list(args)
elif args.command == "sync":