From 27ce7a038bba727f5f91ce537de0133bb82a57c1 Mon Sep 17 00:00:00 2001 From: RapidPoseidon Date: Mon, 13 Jul 2026 12:29:45 +0000 Subject: [PATCH 1/4] docs: update results example to aggregator v4 format Co-Authored-By: elias --- docs/understanding_the_results.md | 121 +++++++++++++++++------------- 1 file changed, 70 insertions(+), 51 deletions(-) diff --git a/docs/understanding_the_results.md b/docs/understanding_the_results.md index 96340b88c..b83a11e6a 100644 --- a/docs/understanding_the_results.md +++ b/docs/understanding_the_results.md @@ -7,79 +7,89 @@ Here's an example of the results you might receive when running a COMPARE task ( ```json { "info": { - "createdAt": "2025-02-11T07:31:59.353232+00:00", - "version": "3.0.0" - }, - "summary": { - "A_wins_total": 0, - "B_wins_total": 1 + "createdAt": "2026-07-13T11:07:53.577348+00:00", + "version": "4.0.0", + "orderType": "Compare", + "name": "Example Image Comparison", + "instruction": "Which image shows: 'A small blue book sitting on a large red book.'?" }, "results": [ { - "context": "A small blue book sitting on a large red book.", "winner_index": 1, - "winner": "dalle-3_37_2.jpg", + "winner": "https://assets.rapidata.ai/flux-1-pro_37_0.jpg", + "assetUrls": { + "https://assets.rapidata.ai/midjourney-5.2_37_3.jpg": "https://assets.rapidata.ai/f8bb5e14-fbba-4f8a-a3dd-88530b5eae48.jpg", + "https://assets.rapidata.ai/flux-1-pro_37_0.jpg": "https://assets.rapidata.ai/e8563fc4-a452-4557-9ab2-d0f06d09ccd7.jpg" + }, "aggregatedResults": { - "aurora-20-1-25_37_4.png": 0, - "dalle-3_37_2.jpg": 3 + "https://assets.rapidata.ai/midjourney-5.2_37_3.jpg": 0, + "https://assets.rapidata.ai/flux-1-pro_37_0.jpg": 3 }, "aggregatedResultsRatios": { - "aurora-20-1-25_37_4.png": 0.0, - "dalle-3_37_2.jpg": 1.0 + "https://assets.rapidata.ai/midjourney-5.2_37_3.jpg": 0.0, + "https://assets.rapidata.ai/flux-1-pro_37_0.jpg": 1.0 }, "summedUserScores": { - "aurora-20-1-25_37_4.png": 0.0, - "dalle-3_37_2.jpg": 1.196 + "https://assets.rapidata.ai/midjourney-5.2_37_3.jpg": 0.0, + "https://assets.rapidata.ai/flux-1-pro_37_0.jpg": 2.552 }, "summedUserScoresRatios": { - "aurora-20-1-25_37_4.png": 0.0, - "dalle-3_37_2.jpg": 1.0 + "https://assets.rapidata.ai/midjourney-5.2_37_3.jpg": 0.0, + "https://assets.rapidata.ai/flux-1-pro_37_0.jpg": 1.0 }, "detailedResults": [ { - "votedFor": "dalle-3_37_2.jpg", + "votedFor": "https://assets.rapidata.ai/flux-1-pro_37_0.jpg", "userDetails": { - "country": "BY", - "language": "ru", + "country": "EG", + "language": "ar", "userScores": { - "global": 0.4469 + "global": 0.845 }, - "demographics": {} + "demographics": { + "age": "18-29", + "gender": "Other", + "occupation": "Not currently working" + } } }, { - "votedFor": "dalle-3_37_2.jpg", + "votedFor": "https://assets.rapidata.ai/flux-1-pro_37_0.jpg", "userDetails": { - "country": "LY", - "language": "ar", + "country": "BD", + "language": "en", "userScores": { - "global": 0.3923 + "global": 0.8349 }, "demographics": { - "age": "0-17", + "age": "65+", "gender": "Other", - "occupation": "Other Employment" + "occupation": "Retired" } } }, { - "votedFor": "dalle-3_37_2.jpg", + "votedFor": "https://assets.rapidata.ai/flux-1-pro_37_0.jpg", "userDetails": { - "country": "BY", - "language": "ru", + "country": "ES", + "language": "es", "userScores": { - "global": 0.3568 + "global": 0.8721 }, "demographics": { "age": "0-17", - "gender": "Other", - "occupation": "Healthcare" + "gender": "Female", + "occupation": "Not currently working" } } } ] } - ] + ], + "summary": { + "A_wins_total": 0, + "B_wins_total": 1 + } } ``` @@ -88,46 +98,51 @@ Here's an example of the results you might receive when running a COMPARE task ( 1. `info` - `createdAt`: The timestamp indicating when the results overview was generated, in UTC time. - `version`: The version of the aggregator system that produced the results. + - `orderType`: The type of task that was run (e.g. `Compare`, `Classify`). + - `name`: The name given to the job or order. + - `instruction`: The instruction that was shown to the labelers. -2. `summary` - - `A_wins_total`: The total number of comparisons won by option A (index 0) across all pairs - - `B_wins_total`: The total number of comparisons won by option B (index 1) across all pairs - -3. `results`: This section contains the actual comparison data collected from the labelers. For comparison jobs, each item includes: +2. `results`: This section contains the actual comparison data collected from the labelers. For comparison jobs, each item includes: - - `context`: The prompt or description provided for the comparison task - `winner_index`: Index of the winning option (0 for first option, 1 for second option) - - `winner`: Filename or identifier of the winning option - + - `winner`: URL of the winning option + - `assetUrls`: Maps each option to the URL under which the asset is hosted by Rapidata — the exact file that was shown to the labelers. + ```json + "assetUrls": { + "https://assets.rapidata.ai/midjourney-5.2_37_3.jpg": "https://assets.rapidata.ai/f8bb5e14-fbba-4f8a-a3dd-88530b5eae48.jpg", + "https://assets.rapidata.ai/flux-1-pro_37_0.jpg": "https://assets.rapidata.ai/e8563fc4-a452-4557-9ab2-d0f06d09ccd7.jpg" + } + ``` + - `aggregatedResults`: The total number of responses each option received for this specific comparison. ```json "aggregatedResults": { - "aurora-20-1-25_37_4.png": 0, - "dalle-3_37_2.jpg": 3 + "https://assets.rapidata.ai/midjourney-5.2_37_3.jpg": 0, + "https://assets.rapidata.ai/flux-1-pro_37_0.jpg": 3 } ``` - `aggregatedResultsRatios`: The proportion of responses each option received, calculated as the number of responses for the option divided by the total number of responses. ```json "aggregatedResultsRatios": { - "aurora-20-1-25_37_4.png": 0.0, - "dalle-3_37_2.jpg": 1.0 + "https://assets.rapidata.ai/midjourney-5.2_37_3.jpg": 0.0, + "https://assets.rapidata.ai/flux-1-pro_37_0.jpg": 1.0 } ``` - `summedUserScores`: The sum of the labelers' global userScore values for each option. This metric accounts for the reliability of each labeler's response. ```json "summedUserScores": { - "aurora-20-1-25_37_4.png": 0.0, - "dalle-3_37_2.jpg": 1.196 + "https://assets.rapidata.ai/midjourney-5.2_37_3.jpg": 0.0, + "https://assets.rapidata.ai/flux-1-pro_37_0.jpg": 2.552 } ``` - `summedUserScoresRatios`: The proportion of the summed global userScores for each option, providing a weighted ratio based on labeler reliability. ```json "summedUserScoresRatios": { - "aurora-20-1-25_37_4.png": 0.0, - "dalle-3_37_2.jpg": 1.0 + "https://assets.rapidata.ai/midjourney-5.2_37_3.jpg": 0.0, + "https://assets.rapidata.ai/flux-1-pro_37_0.jpg": 1.0 } ``` @@ -138,7 +153,11 @@ Here's an example of the results you might receive when running a COMPARE task ( - `language`: Language in which the labeler viewed the task - `userScores`: A score representing the labeler's reliability across different dimensions - `global`: The global userScore of the labeler, which is a measure of their overall reliability - - `demographics`: Demographic attributes collected for the labeler, keyed by attribute name (e.g. `age`, `gender`, `occupation`). + - `demographics`: Demographic attributes collected for the labeler, keyed by attribute name (e.g. `age`, `gender`, `occupation`). May be empty if no demographic data was collected for the labeler. + +3. `summary` + - `A_wins_total`: The total number of comparisons won by option A (index 0) across all pairs + - `B_wins_total`: The total number of comparisons won by option B (index 1) across all pairs ## Understanding the User Scores From fa6f1be5fff0a9e237de3da27af3fe5b32ab95df Mon Sep 17 00:00:00 2001 From: RapidPoseidon Date: Mon, 13 Jul 2026 12:40:39 +0000 Subject: [PATCH 2/4] docs: use bare filenames instead of CDN urls in results example Co-Authored-By: elias --- docs/understanding_the_results.md | 50 +++++++++++++++---------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/docs/understanding_the_results.md b/docs/understanding_the_results.md index b83a11e6a..5d2ea15a3 100644 --- a/docs/understanding_the_results.md +++ b/docs/understanding_the_results.md @@ -16,30 +16,30 @@ Here's an example of the results you might receive when running a COMPARE task ( "results": [ { "winner_index": 1, - "winner": "https://assets.rapidata.ai/flux-1-pro_37_0.jpg", + "winner": "flux-1-pro_37_0.jpg", "assetUrls": { - "https://assets.rapidata.ai/midjourney-5.2_37_3.jpg": "https://assets.rapidata.ai/f8bb5e14-fbba-4f8a-a3dd-88530b5eae48.jpg", - "https://assets.rapidata.ai/flux-1-pro_37_0.jpg": "https://assets.rapidata.ai/e8563fc4-a452-4557-9ab2-d0f06d09ccd7.jpg" + "midjourney-5.2_37_3.jpg": "https://assets.rapidata.ai/f8bb5e14-fbba-4f8a-a3dd-88530b5eae48.jpg", + "flux-1-pro_37_0.jpg": "https://assets.rapidata.ai/e8563fc4-a452-4557-9ab2-d0f06d09ccd7.jpg" }, "aggregatedResults": { - "https://assets.rapidata.ai/midjourney-5.2_37_3.jpg": 0, - "https://assets.rapidata.ai/flux-1-pro_37_0.jpg": 3 + "midjourney-5.2_37_3.jpg": 0, + "flux-1-pro_37_0.jpg": 3 }, "aggregatedResultsRatios": { - "https://assets.rapidata.ai/midjourney-5.2_37_3.jpg": 0.0, - "https://assets.rapidata.ai/flux-1-pro_37_0.jpg": 1.0 + "midjourney-5.2_37_3.jpg": 0.0, + "flux-1-pro_37_0.jpg": 1.0 }, "summedUserScores": { - "https://assets.rapidata.ai/midjourney-5.2_37_3.jpg": 0.0, - "https://assets.rapidata.ai/flux-1-pro_37_0.jpg": 2.552 + "midjourney-5.2_37_3.jpg": 0.0, + "flux-1-pro_37_0.jpg": 2.552 }, "summedUserScoresRatios": { - "https://assets.rapidata.ai/midjourney-5.2_37_3.jpg": 0.0, - "https://assets.rapidata.ai/flux-1-pro_37_0.jpg": 1.0 + "midjourney-5.2_37_3.jpg": 0.0, + "flux-1-pro_37_0.jpg": 1.0 }, "detailedResults": [ { - "votedFor": "https://assets.rapidata.ai/flux-1-pro_37_0.jpg", + "votedFor": "flux-1-pro_37_0.jpg", "userDetails": { "country": "EG", "language": "ar", @@ -54,7 +54,7 @@ Here's an example of the results you might receive when running a COMPARE task ( } }, { - "votedFor": "https://assets.rapidata.ai/flux-1-pro_37_0.jpg", + "votedFor": "flux-1-pro_37_0.jpg", "userDetails": { "country": "BD", "language": "en", @@ -69,7 +69,7 @@ Here's an example of the results you might receive when running a COMPARE task ( } }, { - "votedFor": "https://assets.rapidata.ai/flux-1-pro_37_0.jpg", + "votedFor": "flux-1-pro_37_0.jpg", "userDetails": { "country": "ES", "language": "es", @@ -105,44 +105,44 @@ Here's an example of the results you might receive when running a COMPARE task ( 2. `results`: This section contains the actual comparison data collected from the labelers. For comparison jobs, each item includes: - `winner_index`: Index of the winning option (0 for first option, 1 for second option) - - `winner`: URL of the winning option + - `winner`: Filename or identifier of the winning option - `assetUrls`: Maps each option to the URL under which the asset is hosted by Rapidata — the exact file that was shown to the labelers. ```json "assetUrls": { - "https://assets.rapidata.ai/midjourney-5.2_37_3.jpg": "https://assets.rapidata.ai/f8bb5e14-fbba-4f8a-a3dd-88530b5eae48.jpg", - "https://assets.rapidata.ai/flux-1-pro_37_0.jpg": "https://assets.rapidata.ai/e8563fc4-a452-4557-9ab2-d0f06d09ccd7.jpg" + "midjourney-5.2_37_3.jpg": "https://assets.rapidata.ai/f8bb5e14-fbba-4f8a-a3dd-88530b5eae48.jpg", + "flux-1-pro_37_0.jpg": "https://assets.rapidata.ai/e8563fc4-a452-4557-9ab2-d0f06d09ccd7.jpg" } ``` - `aggregatedResults`: The total number of responses each option received for this specific comparison. ```json "aggregatedResults": { - "https://assets.rapidata.ai/midjourney-5.2_37_3.jpg": 0, - "https://assets.rapidata.ai/flux-1-pro_37_0.jpg": 3 + "midjourney-5.2_37_3.jpg": 0, + "flux-1-pro_37_0.jpg": 3 } ``` - `aggregatedResultsRatios`: The proportion of responses each option received, calculated as the number of responses for the option divided by the total number of responses. ```json "aggregatedResultsRatios": { - "https://assets.rapidata.ai/midjourney-5.2_37_3.jpg": 0.0, - "https://assets.rapidata.ai/flux-1-pro_37_0.jpg": 1.0 + "midjourney-5.2_37_3.jpg": 0.0, + "flux-1-pro_37_0.jpg": 1.0 } ``` - `summedUserScores`: The sum of the labelers' global userScore values for each option. This metric accounts for the reliability of each labeler's response. ```json "summedUserScores": { - "https://assets.rapidata.ai/midjourney-5.2_37_3.jpg": 0.0, - "https://assets.rapidata.ai/flux-1-pro_37_0.jpg": 2.552 + "midjourney-5.2_37_3.jpg": 0.0, + "flux-1-pro_37_0.jpg": 2.552 } ``` - `summedUserScoresRatios`: The proportion of the summed global userScores for each option, providing a weighted ratio based on labeler reliability. ```json "summedUserScoresRatios": { - "https://assets.rapidata.ai/midjourney-5.2_37_3.jpg": 0.0, - "https://assets.rapidata.ai/flux-1-pro_37_0.jpg": 1.0 + "midjourney-5.2_37_3.jpg": 0.0, + "flux-1-pro_37_0.jpg": 1.0 } ``` From 27ce69c284dd6456d5f54cfa5445f0f0f531e613 Mon Sep 17 00:00:00 2001 From: RapidPoseidon Date: Mon, 13 Jul 2026 13:36:06 +0000 Subject: [PATCH 3/4] docs: rename orderType field to type in results example Co-Authored-By: elias --- docs/understanding_the_results.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/understanding_the_results.md b/docs/understanding_the_results.md index 5d2ea15a3..e48a7ed7c 100644 --- a/docs/understanding_the_results.md +++ b/docs/understanding_the_results.md @@ -9,7 +9,7 @@ Here's an example of the results you might receive when running a COMPARE task ( "info": { "createdAt": "2026-07-13T11:07:53.577348+00:00", "version": "4.0.0", - "orderType": "Compare", + "type": "Compare", "name": "Example Image Comparison", "instruction": "Which image shows: 'A small blue book sitting on a large red book.'?" }, @@ -98,7 +98,7 @@ Here's an example of the results you might receive when running a COMPARE task ( 1. `info` - `createdAt`: The timestamp indicating when the results overview was generated, in UTC time. - `version`: The version of the aggregator system that produced the results. - - `orderType`: The type of task that was run (e.g. `Compare`, `Classify`). + - `type`: The type of task that was run (e.g. `Compare`, `Classify`). - `name`: The name given to the job or order. - `instruction`: The instruction that was shown to the labelers. From 00938d5e31591fe53f244d05879d3ac66600bb37 Mon Sep 17 00:00:00 2001 From: RapidPoseidon Date: Mon, 13 Jul 2026 14:17:23 +0000 Subject: [PATCH 4/4] docs: align results example with quickstart order and clarify assetUrls Co-Authored-By: elias --- docs/understanding_the_results.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/understanding_the_results.md b/docs/understanding_the_results.md index e48a7ed7c..64b625027 100644 --- a/docs/understanding_the_results.md +++ b/docs/understanding_the_results.md @@ -10,11 +10,12 @@ Here's an example of the results you might receive when running a COMPARE task ( "createdAt": "2026-07-13T11:07:53.577348+00:00", "version": "4.0.0", "type": "Compare", - "name": "Example Image Comparison", - "instruction": "Which image shows: 'A small blue book sitting on a large red book.'?" + "name": "Example Image Prompt Alignment", + "instruction": "Which image matches the description better?" }, "results": [ { + "context": "A small blue book sitting on a large red book.", "winner_index": 1, "winner": "flux-1-pro_37_0.jpg", "assetUrls": { @@ -104,9 +105,10 @@ Here's an example of the results you might receive when running a COMPARE task ( 2. `results`: This section contains the actual comparison data collected from the labelers. For comparison jobs, each item includes: + - `context`: The context shown alongside this datapoint (present when `contexts` were provided for the job) - `winner_index`: Index of the winning option (0 for first option, 1 for second option) - `winner`: Filename or identifier of the winning option - - `assetUrls`: Maps each option to the URL under which the asset is hosted by Rapidata — the exact file that was shown to the labelers. + - `assetUrls`: Maps each option to the URL under which the asset is hosted by Rapidata — the exact file that was shown to the labelers. The hosted files are not encrypted, but are assigned a random UUID name so they can't be accessed by guessing URLs. ```json "assetUrls": { "midjourney-5.2_37_3.jpg": "https://assets.rapidata.ai/f8bb5e14-fbba-4f8a-a3dd-88530b5eae48.jpg",