unique traces

Author: xzrderek

eval_protocol/adapters/langfuse.py

@@ -68,7 +68,7 @@ def get_evaluation_rows(
         from_timestamp: Optional[datetime] = None,
         to_timestamp: Optional[datetime] = None,
         include_tool_calls: bool = True,
-        page_size: int = 30,
+        page_size: int = 30,  # TODO: remove probably
         sleep_between_gets: float = 0.1,
         max_retries: int = 3,
     ) -> List[EvaluationRow]:

eval_protocol/quickstart/llm_judge.py

@@ -36,9 +36,6 @@
         )
     ],
     completion_params=[
-        # {
-        #     "model": "fireworks_ai/accounts/fireworks/models/qwen3-235b-a22b-instruct-2507",
-        # },
         {"model": "gpt-4.1"},
         {
             "max_tokens": 131000,

eval_protocol/quickstart/utils.py

@@ -112,6 +112,7 @@ def split_multi_turn_rows(data: list[EvaluationRow]) -> list[EvaluationRow]:
         List of expanded EvaluationRow objects, one for each assistant message
     """
     expanded_rows = []
+    seen_traces: set[str] = set()
 
     for row in data:
         messages = row.messages
@@ -128,6 +129,12 @@ def split_multi_turn_rows(data: list[EvaluationRow]) -> list[EvaluationRow]:
             messages_before_assistant = messages[:pos]
             assistant_message = messages[pos]
 
+            # In this case, we trace every request, so we need to filter out duplicates
+            curr_trace = "\n".join(serialize_message(m) for m in messages_before_assistant)
+            if curr_trace in seen_traces:
+                continue
+            seen_traces.add(curr_trace)
+
             ground_truth_message = serialize_message(assistant_message)
 
             expanded_rows.append(