diff --git a/docs.json b/docs.json index a87eb70f66..f32785917f 100644 --- a/docs.json +++ b/docs.json @@ -311,6 +311,16 @@ "models/sweeps/existing-project" ] }, + { + "group" : "Eval Tables", + "pages" : [ + "models/evaltables", + "models/evaltables/create-an-evaluation-table", + "models/evaltables/visualize-evaluation-tables", + "models/evaltables/compare-runs" + ], + "tag" : "Preview" + }, { "group": "Tables", "pages": [ diff --git a/images/evaltables/compare_runs.png b/images/evaltables/compare_runs.png new file mode 100644 index 0000000000..a761d3ec63 Binary files /dev/null and b/images/evaltables/compare_runs.png differ diff --git a/images/evaltables/compare_runs_labled.png b/images/evaltables/compare_runs_labled.png new file mode 100644 index 0000000000..7f94d63ff2 Binary files /dev/null and b/images/evaltables/compare_runs_labled.png differ diff --git a/images/evaltables/eval_column_options.png b/images/evaltables/eval_column_options.png new file mode 100644 index 0000000000..3546064b86 Binary files /dev/null and b/images/evaltables/eval_column_options.png differ diff --git a/images/evaltables/filter.png b/images/evaltables/filter.png new file mode 100644 index 0000000000..9411c67fa5 Binary files /dev/null and b/images/evaltables/filter.png differ diff --git a/images/evaltables/overview_basic_compare_two_runs.png b/images/evaltables/overview_basic_compare_two_runs.png new file mode 100644 index 0000000000..e5e36e05f2 Binary files /dev/null and b/images/evaltables/overview_basic_compare_two_runs.png differ diff --git a/images/evaltables/overview_basic_compare_two_runs_delta_labled.png b/images/evaltables/overview_basic_compare_two_runs_delta_labled.png new file mode 100644 index 0000000000..7d2727b2e9 Binary files /dev/null and b/images/evaltables/overview_basic_compare_two_runs_delta_labled.png differ diff --git a/images/evaltables/overview_basic_compare_two_runs_labled.png b/images/evaltables/overview_basic_compare_two_runs_labled.png new file mode 100644 index 0000000000..216eae0bda Binary files /dev/null and b/images/evaltables/overview_basic_compare_two_runs_labled.png differ diff --git a/images/evaltables/overview_basic_compare_two_runs_landing_page.png b/images/evaltables/overview_basic_compare_two_runs_landing_page.png new file mode 100644 index 0000000000..307986834c Binary files /dev/null and b/images/evaltables/overview_basic_compare_two_runs_landing_page.png differ diff --git a/images/evaltables/overview_basic_compare_two_runs_landing_page_notlabled.png b/images/evaltables/overview_basic_compare_two_runs_landing_page_notlabled.png new file mode 100644 index 0000000000..54452a1a27 Binary files /dev/null and b/images/evaltables/overview_basic_compare_two_runs_landing_page_notlabled.png differ diff --git a/images/evaltables/single_run_eval_table.png b/images/evaltables/single_run_eval_table.png new file mode 100644 index 0000000000..3efc935702 Binary files /dev/null and b/images/evaltables/single_run_eval_table.png differ diff --git a/images/evaltables/single_run_eval_table_labled.png b/images/evaltables/single_run_eval_table_labled.png new file mode 100644 index 0000000000..9f2f0e22a9 Binary files /dev/null and b/images/evaltables/single_run_eval_table_labled.png differ diff --git a/images/evaltables/single_run_eval_table_panel.png b/images/evaltables/single_run_eval_table_panel.png new file mode 100644 index 0000000000..8a5b15516a Binary files /dev/null and b/images/evaltables/single_run_eval_table_panel.png differ diff --git a/models/evaltables.mdx b/models/evaltables.mdx new file mode 100644 index 0000000000..777e06c2a6 --- /dev/null +++ b/models/evaltables.mdx @@ -0,0 +1,32 @@ +--- +title: Eval Tables +description: "Learn how to create, compare, and visualize eval tables in W&B." +--- + +Eval Tables help you compare model outputs, scores, and metrics across runs at the individual-example level. Use an Eval Table to compare model versions or training steps, review aggregate scores, and investigate the examples behind changes in model performance. + +The following image shows an Eval Table called `"validation_prediction_eval"` that compares two runs `"summer-butterfly-9"` and `"gentle-flower-8"`: + + + Evaluation table view + + +An Eval Table panel contains three sections: + +1. Run comparison selector: Select the [runs that you want to compare](/models/evaltables/compare-runs). +2. Aggregate scores: Review aggregate scores for the selected runs and compare the differences between them. For more information, see [View aggregate scores](/models/evaltables/compare-runs#aggregate-scores). +3. Dataset: Compare the inputs, outputs, and scores for each example across the selected runs. + +The following image highlights each section of the panel: + + + Evaluation table view + + +Create an Eval Table with the `EvalTable` class from the W&B Python SDK. You can also convert an existing [W&B Table](/models/tables/) to an Eval Table. + + +Convert existing W&B Tables to Eval Tables to improve rendering performance and access additional comparison features. For instructions, see [Convert a W&B Table to an Eval Table](/models/evaltables/create-an-evaluation-table#convert-a-w&b-table-to-an-eval-table). + + +To create your first Eval Table, see [Create an evaluation table](/models/evaltables/create-an-evaluation-table/). \ No newline at end of file diff --git a/models/evaltables/compare-runs.mdx b/models/evaltables/compare-runs.mdx new file mode 100644 index 0000000000..0b9ae7e9b5 --- /dev/null +++ b/models/evaltables/compare-runs.mdx @@ -0,0 +1,69 @@ +--- +title: Compare runs with eval tables +--- + +Use Eval Tables to compare inputs, outputs, and scores across multiple runs. W&B aligns corresponding examples and calculates aggregate scores and score differences for the selected runs. + +To compare runs: + +1. Navigate to your project workspace. +2. Scroll to the Evaluation Tables panel. +3. Select the **+** button. +4. Select the run that you want to add. + +W&B groups columns that have the same role and name across the selected runs. For example, if two runs use the same `input_columns`, W&B displays those columns together in the **Inputs** section. W&B similarly groups shared `output_columns` and `score_columns` in the **Outputs** and **Scores** sections. + +The following image shows an Eval Table that compares the `summer-butterfly-9` and `gentle-flower-8` runs: + + + Evaluation table view + + +## Set the reference run + +When you compare multiple runs, W&B designates one run as the reference run. W&B uses the reference run as the baseline when it calculates score deltas. + +By default, the leftmost run is the reference run. To choose a different reference run, hover over the run, open its menu, and select **Set as reference**. + + +## View aggregate scores + +W&B calculates an aggregate value for each score column in each selected run. The calculation depends on the score's data type: + +| Type of score | Aggregate value | +| --- | --- | +| Boolean | Count and fraction of `true` values | +| Numeric | Mean | +| String or categorical | No scalar aggregate | +| Null | Excluded from calculation | + +The following image highlights the aggregate values for the correct and confidence scores: + + + Evaluation table view + + +## Compare score deltas + +For each score column, W&B calculates the difference between the reference run and every other selected run. + +W&B calculates each delta as: + +comparison run value - reference run value + +For example, suppose `summer-butterfly-9` is the reference run and `gentle-flower-8` is the comparison run. W&B calculates the confidence delta as follows: + +| `summer-butterfly-9` | `gentle-flower-8` | delta | +| --- | --- | --- | +| 0.43 | 0.62 | +0.19 | +| 0.92 | 0.86 | -0.06 | +| 0.97 | 0.85 | -0.12 | + + +A positive delta means that the comparison run has a higher value than the reference run. A negative delta means that it has a lower value. + +The following image highlights the score deltas: + + + Evaluation table view + \ No newline at end of file diff --git a/models/evaltables/create-an-evaluation-table.mdx b/models/evaltables/create-an-evaluation-table.mdx new file mode 100644 index 0000000000..620cadea1e --- /dev/null +++ b/models/evaltables/create-an-evaluation-table.mdx @@ -0,0 +1,82 @@ +--- +title: Create an eval table +description: "Learn how to create an evaluation table in W&B." +--- + +Create an Eval Table using the W&B Python SDK, or convert an existing [W&B Table](/models/tables/) with ARIA. + +## Create an eval table + +Use `wandb.EvalTable` class to create an Eval Table. + +For example, suppose you have the following pandas DataFrame: + +```python +import pandas as pd +import wandb + +df = pd.DataFrame( + [ + { + "image_id": "img_001", + "true_label": "cat", + "predicted_label": "cat", + "confidence": 0.97, + "correct": True, + }, + { + "image_id": "img_002", + "true_label": "dog", + "predicted_label": "cat", + "confidence": 0.72, + "correct": False, + }, + { + "image_id": "img_003", + "true_label": "car", + "predicted_label": "car", + "confidence": 0.89, + "correct": True, + }, + ] +) +``` + +{/* ```python +with wandb.init(project="classifier-table-demo") as run: + table = wandb.Table(dataframe=df) + run.log({"validation_predictions": table}) +``` */} + +Create an Eval Table by passing the DataFrame to `wandb.EvalTable`. Use the `input_columns`, `output_columns`, and `score_columns` arguments to identify the role of each column. + +The following example uses: + +- `image_id` as the input +- `predicted_label` as the output +- `correct` and `confidence` as scores + +```python +with wandb.init(project="classifier-eval-table-demo") as run: + eval_table = wandb.EvalTable( + dataframe=df, + input_columns=["image_id"], + output_columns=["predicted_label"], + score_columns=["correct", "confidence"], + ) + run.log({"validation_predictions_eval": eval_table}) +``` + + +## Convert a W&B Table to an Eval Table + +Use [ARIA](/aria/) to convert an existing W&B Table to an Eval Table: + +1. Navigate to the project that contains the W&B Table. +2. Select the blue circle in the upper-right corner of the page to open ARIA. +3. Enter `/convert-eval-table` in the ARIA conversation. +4. Select Send, represented by the upward-pointing arrow in the lower-right corner of the conversation. + + +Converting a W&B Table does not modify the original table. + \ No newline at end of file diff --git a/models/evaltables/visualize-evaluation-tables.mdx b/models/evaltables/visualize-evaluation-tables.mdx new file mode 100644 index 0000000000..b1235eab05 --- /dev/null +++ b/models/evaltables/visualize-evaluation-tables.mdx @@ -0,0 +1,146 @@ +--- +title: View eval tables +--- + +When you create an Eval Table, specify which columns contain inputs, outputs, and scores. W&B groups the columns into corresponding sections in the Eval Table. + +The following example shows the basic structure of an EvalTable object: + +```python +import wandb + +wandb.EvalTable( + input_columns=["column1", "column2"], + output_columns=["output_column1", "output_column2"], + score_columns=["score_column1", "score_column2"], +) +``` + +Pass one or more column names to each of the following arguments: + +- `input_columns`: Data provided to the model, such as an image, prompt, or ground-truth label. +- `output_columns`: Values produced by the model, such as a prediction or generated response. +- `score_columns`: Metrics or other values used to evaluate the output. + +W&B displays these columns in the Inputs, Outputs, and Scores sections of the Eval Table. + + +The examples on this page use code from [Create an evaluation table](/models/evaltables/create-an-evaluation-table). Expand **View code** to view or copy the complete example. + + +```python +import pandas as pd +import wandb + +df = pd.DataFrame( + [ + { + "image_id": "img_001", + "true_label": "cat", + "predicted_label": "cat", + "confidence": 0.97, + "correct": True, + }, + { + "image_id": "img_002", + "true_label": "dog", + "predicted_label": "cat", + "confidence": 0.72, + "correct": False, + }, + { + "image_id": "img_003", + "true_label": "car", + "predicted_label": "car", + "confidence": 0.89, + "correct": True, + }, + ] +) + +df["correct"] = df["correct"].astype(object) # preserves native Python bools + +with wandb.init(project="classifier-eval-table-demo") as run: + eval_table = wandb.EvalTable( + dataframe=df, + input_columns=["image_id", "true_label"], + output_columns=["predicted_label"], + score_columns=["correct", "confidence"], + ) + run.log({"validation_predictions_eval": eval_table}) +``` + + + +For example, consider the code example from [Create an evaluation table](/models/evaltables/create-an-evaluation-table): + +```python +import wandb + +with wandb.init(project="classifier-eval-table-demo") as run: + eval_table = wandb.EvalTable( + dataframe=df, + input_columns=["image_id", "true_label"], + output_columns=["predicted_label"], + score_columns=["correct", "confidence"], + ) + run.log({"validation_predictions_eval": eval_table}) +``` + +In that example: + +- The **Inputs** section contains the `image_id` and `true_label` columns. +- The **Outputs** section contains the `predicted_label` column. +- The **Scores** section contains the `correct` and `confidence` columns. + +The following image highlights these sections: + + + Evaluation table view + + +## Detail view + +Select a row to open a detailed view of that example. The detail view shows the example's inputs, outputs, and scores. + +The following image shows the detail view for the first row in the Eval Table: + + + Evaluation table example details + + +Use the detail view to inspect examples when the table contains many columns or when you compare results across multiple runs. + +Select the up or down arrow above the **Inputs** section to move between examples in the current table view. + +## Filter data + +Use filters to display only rows that match specific conditions. + +To add a filter: + +Select Filter above the table. +Select the column to filter. +Select an operator. +Specify a value. + + + Evaluation table filter + + +You can apply multiple filters. To remove a filter, select the X next to it. + +## Show or hide columns + +Use the **Columns** menu to control which columns appear in the Eval Table. The available columns depend on the data logged to the table. + +To show or hide a column: + +1. Select **Columns** above the table. +2. Select a column to show it, or clear the column to hide it. + +The following image shows a Columns menu in which all available columns are selected: + + + Evaluation table columns + \ No newline at end of file diff --git a/models/tables/evaluate-models.mdx b/models/tables/evaluate-models.mdx index f8f7779dd9..539be7bb75 100644 --- a/models/tables/evaluate-models.mdx +++ b/models/tables/evaluate-models.mdx @@ -170,7 +170,7 @@ run.log({"evaluation_results": eval_table}) ### Advanced table workflows #### Compare multiple models -Log evaluation tables from different models to the same key for direct comparison: +Log eval tables from different models to the same key for direct comparison: ```python # Model A evaluation