openapi.yaml

openapi: 3.1.0
info:
  title: Scorecard API
  description: REST API for Scorecard
  version: 1.0.0
servers:
  - url: https://api2.scorecard.io/api/v2
security:
  - ApiKeyAuth: []
paths:
  /projects:
    post:
      operationId: createProject
      summary: Create Project
      description: Create a new Project.
      requestBody:
        content:
          application/json:
            schema:
              type: object
              properties:
                name:
                  type: string
                  description: The name of the Project.
                description:
                  type: string
                  description: The description of the Project.
              required:
                - name
                - description
            examples:
              Create a new project:
                value:
                  name: My Project
                  description: This is a test project
                summary: Create a new project
                description: Request to create a new project with a name and description.
      responses:
        "201":
          description: Project created successfully
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/Project"
              examples:
                Created project response:
                  value:
                    id: "314"
                    name: My Project
                    description: This is a test project
                  summary: Created project response
                  description: Response after successfully creating a project.
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: >-
            import Scorecard from 'scorecard-ai';


            const client = new Scorecard({
              apiKey: 'My API Key',
            });


            const project = await client.projects.create({ description: 'This is a test project', name: 'My
            Project' });


            console.log(project.id);
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            project = client.projects.create(
                description="This is a test project",
                name="My Project",
            )
            print(project.id)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/projects \
                -H 'Content-Type: application/json' \
                -H "Authorization: Bearer $SCORECARD_API_KEY" \
                -d '{
                      "description": "This is a test project",
                      "name": "My Project"
                    }'
    get:
      operationId: listProjects
      summary: List Projects
      description: >-
        Retrieve a paginated list of all Projects. Projects are ordered by creation date, with oldest Projects
        first.
      parameters:
        - in: query
          name: limit
          description: Maximum number of items to return (1-100). Use with `cursor` for pagination through large sets.
          schema:
            type: integer
            exclusiveMinimum: 0
            default: 20
            example: 20
        - in: query
          name: cursor
          description: >-
            Cursor for pagination. Pass the `nextCursor` from the previous response to get the next page of
            results.
          schema:
            type: string
            example: "123"
      responses:
        "200":
          description: Successfully retrieved list of Projects.
          content:
            application/json:
              schema:
                type: object
                properties:
                  data:
                    type: array
                    items:
                      $ref: "#/components/schemas/Project"
                  nextCursor:
                    type:
                      - string
                      - "null"
                  hasMore:
                    type: boolean
                  total:
                    type: integer
                    minimum: 0
                required:
                  - data
                  - nextCursor
                  - hasMore
              examples:
                Project list with pagination:
                  value:
                    data:
                      - id: "123"
                        name: Q&A Chatbot
                        description: Chatbot for answering questions about the company.
                      - id: "124"
                        name: Summarizer (Europe)
                        description: Summarizer for documents in the Europe region.
                    nextCursor: "125"
                    hasMore: true
                  summary: Project list with pagination
                  description: Example response showing a list of two Projects with pagination information.
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: |-
            import Scorecard from 'scorecard-ai';

            const client = new Scorecard({
              apiKey: 'My API Key',
            });

            // Automatically fetches more pages as needed.
            for await (const project of client.projects.list()) {
              console.log(project.id);
            }
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            page = client.projects.list()
            page = page.data[0]
            print(page.id)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/projects \
                -H "Authorization: Bearer $SCORECARD_API_KEY"
  /projects/{projectId}/testsets:
    get:
      operationId: listTestsets
      summary: List Testsets in Project
      description: Retrieve a paginated list of Testsets belonging to a Project.
      parameters:
        - in: path
          name: projectId
          description: The ID of the Project.
          schema:
            type: string
            example: "314"
          required: true
        - in: query
          name: limit
          description: Maximum number of items to return (1-100). Use with `cursor` for pagination through large sets.
          schema:
            type: integer
            exclusiveMinimum: 0
            default: 20
            example: 20
        - in: query
          name: cursor
          description: >-
            Cursor for pagination. Pass the `nextCursor` from the previous response to get the next page of
            results.
          schema:
            type: string
            example: "123"
      responses:
        "200":
          description: Successfully retrieved list of Testsets.
          content:
            application/json:
              schema:
                type: object
                properties:
                  data:
                    type: array
                    items:
                      $ref: "#/components/schemas/Testset"
                  nextCursor:
                    type:
                      - string
                      - "null"
                  hasMore:
                    type: boolean
                  total:
                    type: integer
                    minimum: 0
                required:
                  - data
                  - nextCursor
                  - hasMore
              examples:
                Testset list with fields:
                  value:
                    data:
                      - id: "246"
                        name: Long Context Q&A
                        description: Testset for long context Q&A chatbot.
                        jsonSchema:
                          type: object
                          properties:
                            question:
                              type: string
                            idealAnswer:
                              type: string
                            provenance:
                              type: string
                            geo:
                              type: string
                        fieldMapping:
                          inputs:
                            - question
                          expected:
                            - idealAnswer
                          metadata:
                            - provenance
                            - geo
                    nextCursor: "247"
                    hasMore: true
                  summary: Testset list with fields
                  description: >-
                    Example response showing a paginated list of Testsets with schema and field mapping
                    details.
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: |-
            import Scorecard from 'scorecard-ai';

            const client = new Scorecard({
              apiKey: 'My API Key',
            });

            // Automatically fetches more pages as needed.
            for await (const testset of client.testsets.list('314')) {
              console.log(testset.id);
            }
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            page = client.testsets.list(
                project_id="314",
            )
            page = page.data[0]
            print(page.id)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/projects/$PROJECT_ID/testsets \
                -H "Authorization: Bearer $SCORECARD_API_KEY"
    post:
      operationId: createTestset
      summary: Create Testset
      description: Create a new Testset for a Project. The Testset will be created in the Project specified in the path.
      parameters:
        - in: path
          name: projectId
          description: The ID of the Project to create the Testset in.
          schema:
            type: string
            example: "314"
          required: true
      requestBody:
        content:
          application/json:
            schema:
              type: object
              properties:
                name:
                  type: string
                  description: The name of the Testset.
                description:
                  type: string
                  description: The description of the Testset.
                jsonSchema:
                  type: object
                  description: The JSON schema for each Testcase in the Testset.
                  additionalProperties: true
                fieldMapping:
                  type: object
                  properties:
                    inputs:
                      type: array
                      items:
                        type: string
                      description: Fields that represent inputs to the AI system.
                    expected:
                      type: array
                      items:
                        type: string
                      description: Fields that represent expected outputs.
                    metadata:
                      type: array
                      items:
                        type: string
                      description: Fields that are not inputs or expected outputs.
                  required:
                    - inputs
                    - expected
                    - metadata
                  description: >-
                    Maps top-level keys of the Testcase schema to their roles (input/expected output).
                    Unmapped fields are treated as metadata.
              required:
                - name
                - description
                - jsonSchema
                - fieldMapping
            examples:
              Create Q&A Testset:
                value:
                  name: Long Context Q&A
                  description: Testset for long context Q&A chatbot.
                  jsonSchema:
                    type: object
                    properties:
                      question:
                        type: string
                      idealAnswer:
                        type: string
                      provenance:
                        type: string
                      geo:
                        type: string
                  fieldMapping:
                    inputs:
                      - question
                    expected:
                      - idealAnswer
                    metadata: []
                summary: Create Q&A Testset
                description: >-
                  Request to create a Testset for evaluating long context Q&A with fields for the question,
                  ideal answer, and metadata like provenance and geographical context.
      responses:
        "201":
          description: Testset created successfully
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/Testset"
              examples:
                Created Testset response:
                  value:
                    id: "246"
                    name: Long Context Q&A
                    description: Testset for long context Q&A chatbot.
                    jsonSchema:
                      type: object
                      properties:
                        question:
                          type: string
                        idealAnswer:
                          type: string
                        provenance:
                          type: string
                        geo:
                          type: string
                    fieldMapping:
                      inputs:
                        - question
                      expected:
                        - idealAnswer
                      metadata:
                        - provenance
                        - geo
                  summary: Created Testset response
                  description: >-
                    Response after successfully creating a Testset, showing the assigned ID and complete
                    schema with automatically populated field mappings.
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: |-
            import Scorecard from 'scorecard-ai';

            const client = new Scorecard({
              apiKey: 'My API Key',
            });

            const testset = await client.testsets.create('314', {
              description: 'Testset for long context Q&A chatbot.',
              fieldMapping: { inputs: ['question'], expected: ['idealAnswer'], metadata: [] },
              jsonSchema: {
                type: 'object',
                properties: {
                  question: { type: 'string' },
                  idealAnswer: { type: 'string' },
                  provenance: { type: 'string' },
                  geo: { type: 'string' },
                },
              },
              name: 'Long Context Q&A',
            });

            console.log(testset.id);
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            testset = client.testsets.create(
                project_id="314",
                description="Testset for long context Q&A chatbot.",
                field_mapping={
                    "inputs": ["question"],
                    "expected": ["idealAnswer"],
                    "metadata": [],
                },
                json_schema={
                    "type": "object",
                    "properties": {
                        "question": {
                            "type": "string"
                        },
                        "idealAnswer": {
                            "type": "string"
                        },
                        "provenance": {
                            "type": "string"
                        },
                        "geo": {
                            "type": "string"
                        },
                    },
                },
                name="Long Context Q&A",
            )
            print(testset.id)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/projects/$PROJECT_ID/testsets \
                -H 'Content-Type: application/json' \
                -H "Authorization: Bearer $SCORECARD_API_KEY" \
                -d '{
                      "description": "Testset for long context Q&A chatbot.",
                      "fieldMapping": {
                        "expected": [
                          "idealAnswer"
                        ],
                        "inputs": [
                          "question"
                        ],
                        "metadata": [
                          "string"
                        ]
                      },
                      "jsonSchema": {
                        "type": "bar",
                        "properties": "bar"
                      },
                      "name": "Long Context Q&A"
                    }'
  /testsets/{testsetId}:
    get:
      operationId: getTestset
      summary: Get Testset
      parameters:
        - in: path
          name: testsetId
          description: The ID of the Testset.
          schema:
            type: string
            example: "246"
          required: true
      responses:
        "200":
          description: Successfully retrieved Testset
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/Testset"
              examples:
                Complete Testset details:
                  value:
                    id: "246"
                    name: Long Context Q&A
                    description: Testset for long context Q&A chatbot.
                    jsonSchema:
                      type: object
                      properties:
                        question:
                          type: string
                        idealAnswer:
                          type: string
                        provenance:
                          type: string
                        geo:
                          type: string
                    fieldMapping:
                      inputs:
                        - question
                      expected:
                        - idealAnswer
                      metadata:
                        - provenance
                        - geo
                  summary: Complete Testset details
                  description: >-
                    Example response showing all details of a Testset including its schema definition and
                    field mappings.
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: |-
            import Scorecard from 'scorecard-ai';

            const client = new Scorecard({
              apiKey: 'My API Key',
            });

            const testset = await client.testsets.get('246');

            console.log(testset.id);
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            testset = client.testsets.get(
                "246",
            )
            print(testset.id)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/testsets/$TESTSET_ID \
                -H "Authorization: Bearer $SCORECARD_API_KEY"
    patch:
      operationId: updateTestset
      summary: Update Testset
      description: >-
        Update a Testset. Only the fields provided in the request body will be updated.

        If a field is provided, the new content will replace the existing content.

        If a field is not provided, the existing content will remain unchanged.


        When updating the schema:

        - If field mappings are not provided and existing mappings reference fields that no longer exist,
        those mappings will be automatically removed

        - To preserve all existing mappings, ensure all referenced fields remain in the updated schema

        - For complete control, provide both schema and fieldMapping when updating the schema
      parameters:
        - in: path
          name: testsetId
          description: The ID of the Testset to update.
          schema:
            type: string
            example: "246"
          required: true
      requestBody:
        content:
          application/json:
            schema:
              type: object
              properties:
                name:
                  type: string
                  description: The name of the Testset.
                description:
                  type: string
                  description: The description of the Testset.
                jsonSchema:
                  type: object
                  description: The JSON schema for each Testcase in the Testset.
                  additionalProperties: true
                fieldMapping:
                  type: object
                  properties:
                    inputs:
                      type: array
                      items:
                        type: string
                      description: Fields that represent inputs to the AI system.
                    expected:
                      type: array
                      items:
                        type: string
                      description: Fields that represent expected outputs.
                    metadata:
                      type: array
                      items:
                        type: string
                      description: Fields that are not inputs or expected outputs.
                  required:
                    - inputs
                    - expected
                    - metadata
                  description: >-
                    Maps top-level keys of the Testcase schema to their roles (input/expected output).
                    Unmapped fields are treated as metadata.
            examples:
              Update metadata only:
                value:
                  name: Updated Q&A Testset
                  description: Updated description for the Q&A Testset.
                summary: Update metadata only
                description: >-
                  Simple metadata update without changing schema or mappings. Updates only the name and
                  description fields while preserving the existing schema and field mappings.
              Remove schema field:
                value:
                  jsonSchema:
                    type: object
                    properties:
                      question:
                        type: string
                      idealAnswer:
                        type: string
                      provenance:
                        type: string
                summary: Remove schema field
                description: >-
                  This request removes the 'geo' field that existed in the original schema, but doesn't
                  explicitly update the field mappings. Scorecard will automatically remove any field mappings
                  that reference deleted fields.
              Full schema revision:
                value:
                  jsonSchema:
                    type: object
                    properties:
                      question:
                        type: string
                      idealAnswer:
                        type: string
                      provenance:
                        type: string
                  fieldMapping:
                    inputs:
                      - question
                    expected:
                      - idealAnswer
                    metadata: []
                summary: Full schema revision
                description: >-
                  Explicit update of both schema and field mappings, allowing complete control over the
                  Testset structure. This example removes the 'geo' field and explicitly updates the field
                  mappings to exclude 'provenance' from metadata.
      responses:
        "200":
          description: Testset updated successfully
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/Testset"
              examples:
                Updated metadata only:
                  value:
                    id: "246"
                    name: Updated Q&A Testset
                    description: Updated description for the Q&A Testset.
                    jsonSchema:
                      type: object
                      properties:
                        question:
                          type: string
                        idealAnswer:
                          type: string
                        provenance:
                          type: string
                        geo:
                          type: string
                    fieldMapping:
                      inputs:
                        - question
                      expected:
                        - idealAnswer
                      metadata:
                        - provenance
                        - geo
                  summary: Updated metadata only
                  description: >-
                    Result after updating only the Testset's name and description. All schema fields and
                    mappings remain unchanged.
                Auto-updated mappings:
                  value:
                    id: "246"
                    name: Long Context Q&A
                    description: Testset for long context Q&A chatbot.
                    jsonSchema:
                      type: object
                      properties:
                        question:
                          type: string
                        idealAnswer:
                          type: string
                        provenance:
                          type: string
                    fieldMapping:
                      inputs:
                        - question
                      expected:
                        - idealAnswer
                      metadata:
                        - provenance
                  summary: Auto-updated mappings
                  description: >-
                    Result after schema update with automatic field mapping cleanup. The 'geo' field has been
                    automatically removed from the metadata mapping since it no longer exists in the schema.
                Custom field mapping:
                  value:
                    id: "246"
                    name: Long Context Q&A
                    description: Testset for long context Q&A chatbot.
                    jsonSchema:
                      type: object
                      properties:
                        question:
                          type: string
                        idealAnswer:
                          type: string
                        provenance:
                          type: string
                    fieldMapping:
                      inputs:
                        - question
                      expected:
                        - idealAnswer
                      metadata: []
                  summary: Custom field mapping
                  description: >-
                    Result after explicit schema and field mapping update. Note that 'provenance' is not
                    included in metadata since it wasn't specified in the request's field mapping.
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: |-
            import Scorecard from 'scorecard-ai';

            const client = new Scorecard({
              apiKey: 'My API Key',
            });

            const testset = await client.testsets.update('246', {
              description: 'Updated description for the Q&A Testset.',
              name: 'Updated Q&A Testset',
            });

            console.log(testset.id);
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            testset = client.testsets.update(
                testset_id="246",
                description="Updated description for the Q&A Testset.",
                name="Updated Q&A Testset",
            )
            print(testset.id)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/testsets/$TESTSET_ID \
                -X PATCH \
                -H "Authorization: Bearer $SCORECARD_API_KEY"
    delete:
      operationId: deleteTestset
      summary: Delete Testset
      parameters:
        - in: path
          name: testsetId
          description: The ID of the Testset to delete.
          schema:
            type: string
            example: "246"
          required: true
      responses:
        "200":
          description: Testset deleted successfully
          content:
            application/json:
              schema:
                type: object
                properties:
                  success:
                    type: boolean
                    description: Whether the deletion was successful.
                required:
                  - success
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: |-
            import Scorecard from 'scorecard-ai';

            const client = new Scorecard({
              apiKey: 'My API Key',
            });

            const testset = await client.testsets.delete('246');

            console.log(testset.success);
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            testset = client.testsets.delete(
                "246",
            )
            print(testset.success)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/testsets/$TESTSET_ID \
                -X DELETE \
                -H "Authorization: Bearer $SCORECARD_API_KEY"
  /testsets/{testsetId}/testcases:
    get:
      operationId: listTestcases
      summary: List Testcases in Testset
      description: Retrieve a paginated list of Testcases belonging to a Testset.
      parameters:
        - in: path
          name: testsetId
          description: The ID of the Testset to list Testcases from.
          schema:
            type: string
            example: "246"
          required: true
        - in: query
          name: limit
          description: Maximum number of items to return (1-100). Use with `cursor` for pagination through large sets.
          schema:
            type: integer
            exclusiveMinimum: 0
            default: 20
            example: 20
        - in: query
          name: cursor
          description: >-
            Cursor for pagination. Pass the `nextCursor` from the previous response to get the next page of
            results.
          schema:
            type: string
            example: "123"
      responses:
        "200":
          description: Successfully retrieved list of Testcases.
          content:
            application/json:
              schema:
                type: object
                properties:
                  data:
                    type: array
                    items:
                      $ref: "#/components/schemas/Testcase"
                  nextCursor:
                    type:
                      - string
                      - "null"
                  hasMore:
                    type: boolean
                  total:
                    type: integer
                    minimum: 0
                required:
                  - data
                  - nextCursor
                  - hasMore
              examples:
                Testcase list with pagination:
                  value:
                    data:
                      - id: "123"
                        testsetId: "246"
                        jsonData:
                          question: What is the capital of France?
                          idealAnswer: Paris
                          provenance: hand_curated
                        inputs:
                          question: What is the capital of France?
                        expected:
                          idealAnswer: Paris
                      - id: "124"
                        testsetId: "246"
                        jsonData:
                          question: What is the largest planet in our solar system?
                          idealAnswer: Jupiter
                          provenance: synthetic
                        inputs:
                          question: What is the largest planet in our solar system?
                        expected:
                          idealAnswer: Jupiter
                      - id: "125"
                        testsetId: "246"
                        jsonData:
                          question: What is the deepest ocean on Earth?
                          provenance: user_feedback
                        inputs:
                          question: What is the deepest ocean on Earth?
                        expected: {}
                        validationErrors:
                          - path: /data
                            message: Required field 'idealAnswer' is missing
                    nextCursor: "126"
                    hasMore: true
                  summary: Testcase list with pagination
                  description: Example response showing a paginated list of Testcases.
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: |-
            import Scorecard from 'scorecard-ai';

            const client = new Scorecard({
              apiKey: 'My API Key',
            });

            // Automatically fetches more pages as needed.
            for await (const testcase of client.testcases.list('246')) {
              console.log(testcase.id);
            }
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            page = client.testcases.list(
                testset_id="246",
            )
            page = page.data[0]
            print(page.id)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/testsets/$TESTSET_ID/testcases \
                -H "Authorization: Bearer $SCORECARD_API_KEY"
    post:
      operationId: createTestcases
      summary: Create multiple Testcases
      description: Create multiple Testcases in the specified Testset.
      parameters:
        - in: path
          name: testsetId
          description: The ID of the Testset to add the Testcases to.
          schema:
            type: string
            example: "246"
          required: true
      requestBody:
        content:
          application/json:
            schema:
              type: object
              properties:
                items:
                  type: array
                  items:
                    type: object
                    properties:
                      jsonData:
                        type: object
                        additionalProperties: true
                        description: The JSON data of the Testcase, which is validated against the Testset's schema.
                    required:
                      - jsonData
                  minItems: 1
                  maxItems: 100
                  description: Testcases to create (max 100).
              required:
                - items
            examples:
              Create multiple Testcases:
                value:
                  items:
                    - jsonData:
                        question: What is the capital of France?
                        idealAnswer: Paris
                        provenance: hand_curated
                    - jsonData:
                        question: What is the largest planet in our solar system?
                        idealAnswer: Jupiter
                        provenance: synthetic
                    - jsonData:
                        question: How many planets are in our solar system?
                        idealAnswer: 8
                        provenance: user_feedback
                summary: Create multiple Testcases
                description: Create multiple Testcases in a single request.
      responses:
        "201":
          description: Testcases created successfully
          content:
            application/json:
              schema:
                type: object
                properties:
                  items:
                    type: array
                    items:
                      $ref: "#/components/schemas/Testcase"
                required:
                  - items
              examples:
                Created Testcases response:
                  value:
                    items:
                      - id: "123"
                        testsetId: "246"
                        jsonData:
                          question: What is the capital of France?
                          idealAnswer: Paris
                          provenance: hand_curated
                        inputs:
                          question: What is the capital of France?
                        expected:
                          idealAnswer: Paris
                      - id: "124"
                        testsetId: "246"
                        jsonData:
                          question: What is the largest planet in our solar system?
                          idealAnswer: Jupiter
                          provenance: synthetic
                        inputs:
                          question: What is the largest planet in our solar system?
                        expected:
                          idealAnswer: Jupiter
                      - id: "125"
                        testsetId: "246"
                        jsonData:
                          question: How many planets are in our solar system?
                          idealAnswer: 8
                          provenance: user_feedback
                        inputs:
                          question: How many planets are in our solar system?
                        expected:
                          idealAnswer: 8
                        validationErrors:
                          - path: /data/idealAnswer
                            message: Expected string, received number
                  summary: Created Testcases
                  description: Example response showing successfully created Testcases.
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: |-
            import Scorecard from 'scorecard-ai';

            const client = new Scorecard({
              apiKey: 'My API Key',
            });

            const testcase = await client.testcases.create('246', {
              items: [
                {
                  jsonData: {
                    question: 'What is the capital of France?',
                    idealAnswer: 'Paris',
                    provenance: 'hand_curated',
                  },
                },
                {
                  jsonData: {
                    question: 'What is the largest planet in our solar system?',
                    idealAnswer: 'Jupiter',
                    provenance: 'synthetic',
                  },
                },
                {
                  jsonData: {
                    question: 'How many planets are in our solar system?',
                    idealAnswer: 8,
                    provenance: 'user_feedback',
                  },
                },
              ],
            });

            console.log(testcase.items);
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            testcase = client.testcases.create(
                testset_id="246",
                items=[{
                    "json_data": {
                        "question": "What is the capital of France?",
                        "idealAnswer": "Paris",
                        "provenance": "hand_curated",
                    }
                }, {
                    "json_data": {
                        "question": "What is the largest planet in our solar system?",
                        "idealAnswer": "Jupiter",
                        "provenance": "synthetic",
                    }
                }, {
                    "json_data": {
                        "question": "How many planets are in our solar system?",
                        "idealAnswer": 8,
                        "provenance": "user_feedback",
                    }
                }],
            )
            print(testcase.items)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/testsets/$TESTSET_ID/testcases \
                -H 'Content-Type: application/json' \
                -H "Authorization: Bearer $SCORECARD_API_KEY" \
                -d '{
                      "items": [
                        {
                          "jsonData": {
                            "question": "bar",
                            "idealAnswer": "bar",
                            "provenance": "bar"
                          }
                        },
                        {
                          "jsonData": {
                            "question": "bar",
                            "idealAnswer": "bar",
                            "provenance": "bar"
                          }
                        },
                        {
                          "jsonData": {
                            "question": "bar",
                            "idealAnswer": "bar",
                            "provenance": "bar"
                          }
                        }
                      ]
                    }'
  /testcases/{testcaseId}:
    get:
      operationId: getTestcase
      summary: Get Testcase
      description: Retrieve a specific Testcase by ID.
      parameters:
        - in: path
          name: testcaseId
          description: The ID of the Testcase to retrieve.
          schema:
            type: string
            example: "248"
          required: true
      responses:
        "200":
          description: Successfully retrieved Testcase.
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/Testcase"
              examples:
                Testcase details:
                  value:
                    id: "248"
                    testsetId: "246"
                    jsonData:
                      question: What is the capital of France?
                      idealAnswer: Paris
                      provenance: hand_curated
                    inputs:
                      question: What is the capital of France?
                    expected:
                      idealAnswer: Paris
                  summary: Testcase details
                  description: Example response showing a Testcase's details.
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: |-
            import Scorecard from 'scorecard-ai';

            const client = new Scorecard({
              apiKey: 'My API Key',
            });

            const testcase = await client.testcases.get('248');

            console.log(testcase.id);
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            testcase = client.testcases.get(
                "248",
            )
            print(testcase.id)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/testcases/$TESTCASE_ID \
                -H "Authorization: Bearer $SCORECARD_API_KEY"
    put:
      operationId: updateTestcase
      summary: Update Testcase
      description: Replace the data of an existing Testcase while keeping its ID.
      parameters:
        - in: path
          name: testcaseId
          description: The ID of the Testcase to update.
          schema:
            type: string
            example: "248"
          required: true
      requestBody:
        content:
          application/json:
            schema:
              type: object
              properties:
                jsonData:
                  type: object
                  additionalProperties: true
                  description: The JSON data of the Testcase, which is validated against the Testset's schema.
              required:
                - jsonData
            examples:
              Update Testcase data:
                value:
                  jsonData:
                    question: What is the capital of France?
                    idealAnswer: Paris is the capital of France
                    provenance: hand_curated
                summary: Update Testcase data
                description: Update the content of a Testcase with improved information.
      responses:
        "200":
          description: Testcase updated successfully
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/Testcase"
              examples:
                Updated Testcase:
                  value:
                    id: "248"
                    testsetId: "246"
                    jsonData:
                      question: What is the capital of France?
                      idealAnswer: Paris is the capital of France
                      provenance: hand_curated
                    inputs:
                      question: What is the capital of France?
                    expected:
                      idealAnswer: Paris is the capital of France
                  summary: Updated Testcase
                  description: Example response showing a successfully updated Testcase.
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: |-
            import Scorecard from 'scorecard-ai';

            const client = new Scorecard({
              apiKey: 'My API Key',
            });

            const testcase = await client.testcases.update('248', {
              jsonData: {
                question: 'What is the capital of France?',
                idealAnswer: 'Paris is the capital of France',
                provenance: 'hand_curated',
              },
            });

            console.log(testcase.id);
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            testcase = client.testcases.update(
                testcase_id="248",
                json_data={
                    "question": "What is the capital of France?",
                    "idealAnswer": "Paris is the capital of France",
                    "provenance": "hand_curated",
                },
            )
            print(testcase.id)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/testcases/$TESTCASE_ID \
                -X PUT \
                -H 'Content-Type: application/json' \
                -H "Authorization: Bearer $SCORECARD_API_KEY" \
                -d '{
                      "jsonData": {
                        "question": "bar",
                        "idealAnswer": "bar",
                        "provenance": "bar"
                      }
                    }'
  /testcases/bulk-delete:
    post:
      operationId: deleteTestcases
      summary: Delete multiple Testcases
      description: Delete multiple Testcases by their IDs.
      requestBody:
        content:
          application/json:
            schema:
              type: object
              properties:
                ids:
                  type: array
                  items:
                    type: string
                  description: IDs of Testcases to delete.
              required:
                - ids
            examples:
              Delete multiple Testcases:
                value:
                  ids:
                    - "123"
                    - "124"
                    - "125"
      responses:
        "200":
          description: Testcases deleted successfully
          content:
            application/json:
              schema:
                type: object
                properties:
                  success:
                    type: boolean
                    description: Whether the deletion was successful.
                required:
                  - success
              examples:
                Delete multiple Testcases:
                  value:
                    success: true
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: |-
            import Scorecard from 'scorecard-ai';

            const client = new Scorecard({
              apiKey: 'My API Key',
            });

            const testcase = await client.testcases.delete({ ids: ['123', '124', '125'] });

            console.log(testcase.success);
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            testcase = client.testcases.delete(
                ids=["123", "124", "125"],
            )
            print(testcase.success)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/testcases/bulk-delete \
                -H 'Content-Type: application/json' \
                -H "Authorization: Bearer $SCORECARD_API_KEY" \
                -d '{
                      "ids": [
                        "123",
                        "124",
                        "125"
                      ]
                    }'
  /projects/{projectId}/metrics:
    get:
      operationId: listMetrics
      summary: List Metrics
      description: >-
        List Metrics configured for the specified Project. Metrics are returned in reverse chronological
        order.
      parameters:
        - in: path
          name: projectId
          description: The ID of the Project to list Metrics for.
          schema:
            type: string
            example: "314"
          required: true
        - in: query
          name: limit
          description: Maximum number of items to return (1-100). Use with `cursor` for pagination through large sets.
          schema:
            type: integer
            exclusiveMinimum: 0
            default: 20
            example: 20
        - in: query
          name: cursor
          description: >-
            Cursor for pagination. Pass the `nextCursor` from the previous response to get the next page of
            results.
          schema:
            type: string
            example: "123"
      responses:
        "200":
          description: Successfully retrieved Metrics.
          content:
            application/json:
              schema:
                type: object
                properties:
                  data:
                    type: array
                    items:
                      $ref: "#/components/schemas/Metric"
                  nextCursor:
                    type:
                      - string
                      - "null"
                  hasMore:
                    type: boolean
                  total:
                    type: integer
                    minimum: 0
                required:
                  - data
                  - nextCursor
                  - hasMore
              examples:
                List Metrics:
                  summary: List Metrics
                  description: Example response showing metrics with different evaluation and output types.
                  value:
                    data:
                      - id: "456"
                        name: Response Accuracy
                        description: Evaluates if the response is factually accurate
                        outputType: boolean
                        evalType: ai
                        guidelines: Check if the response contains factually correct information
                        promptTemplate: "Please evaluate if the following response is factually accurate: {{response}}"
                        evalModelName: gpt-4o
                        temperature: 0.1
                      - id: "457"
                        name: Response Quality
                        description: Human review of quality
                        outputType: float
                        evalType: human
                        guidelines: Rate the coherence of the response (0-1).
                        passingThreshold: 0.8
                    nextCursor: null
                    hasMore: false
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: |-
            import Scorecard from 'scorecard-ai';

            const client = new Scorecard({
              apiKey: 'My API Key',
            });

            // Automatically fetches more pages as needed.
            for await (const metric of client.metrics.list('314')) {
              console.log(metric);
            }
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            page = client.metrics.list(
                project_id="314",
            )
            page = page.data[0]
            print(page)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/projects/$PROJECT_ID/metrics \
                -H "Authorization: Bearer $SCORECARD_API_KEY"
    post:
      operationId: createMetric
      summary: Create Metric
      description: >-
        Create a new Metric for evaluating system outputs. The structure of a metric depends on the evalType
        and outputType of the metric.
      parameters:
        - in: path
          name: projectId
          description: The ID of the Project to create the Metric in.
          schema:
            type: string
            example: "314"
          required: true
      requestBody:
        content:
          application/json:
            schema:
              anyOf:
                - type: object
                  properties:
                    name:
                      type: string
                      description: The name of the Metric.
                    description:
                      type:
                        - string
                        - "null"
                      default: null
                      description: The description of the Metric.
                    evalType:
                      type: string
                      const: ai
                      description: AI-based evaluation type.
                    guidelines:
                      type: string
                      description: Guidelines for AI evaluation on how to score the metric.
                    promptTemplate:
                      type: string
                      description: >-
                        The complete prompt template for AI evaluation. Should include placeholders for
                        dynamic content.
                    evalModelName:
                      type: string
                      default: gpt-4o
                      description: The AI model to use for evaluation.
                    temperature:
                      type: number
                      minimum: 0
                      maximum: 2
                      default: 0
                      description: The temperature for AI evaluation (0-2).
                    outputType:
                      type: string
                      const: int
                      description: Integer output type.
                    passingThreshold:
                      type: integer
                      minimum: 1
                      maximum: 5
                      default: 4
                      description: The threshold for determining pass/fail from integer scores (1-5).
                  required:
                    - name
                    - evalType
                    - promptTemplate
                    - outputType
                  description: A Metric with AI evaluation and integer output.
                  title: AI int metric
                - type: object
                  properties:
                    name:
                      type: string
                      description: The name of the Metric.
                    description:
                      type:
                        - string
                        - "null"
                      default: null
                      description: The description of the Metric.
                    evalType:
                      type: string
                      const: human
                      description: Human-based evaluation type.
                    guidelines:
                      type: string
                      description: Guidelines for human evaluators.
                    outputType:
                      type: string
                      const: int
                      description: Integer output type.
                    passingThreshold:
                      type: integer
                      minimum: 1
                      maximum: 5
                      default: 4
                      description: The threshold for determining pass/fail from integer scores (1-5).
                  required:
                    - name
                    - evalType
                    - outputType
                  description: A Metric with human evaluation and integer output.
                  title: Human int metric
                - type: object
                  properties:
                    name:
                      type: string
                      description: The name of the Metric.
                    description:
                      type:
                        - string
                        - "null"
                      default: null
                      description: The description of the Metric.
                    evalType:
                      type: string
                      const: heuristic
                      description: Heuristic-based evaluation type.
                    guidelines:
                      type: string
                      description: Guidelines for heuristic evaluation logic.
                    outputType:
                      type: string
                      const: int
                      description: Integer output type.
                    passingThreshold:
                      type: integer
                      minimum: 1
                      maximum: 5
                      default: 4
                      description: The threshold for determining pass/fail from integer scores (1-5).
                  required:
                    - name
                    - evalType
                    - outputType
                  description: A Metric with heuristic evaluation and integer output.
                  title: Heuristic int metric
                - type: object
                  properties:
                    name:
                      type: string
                      description: The name of the Metric.
                    description:
                      type:
                        - string
                        - "null"
                      default: null
                      description: The description of the Metric.
                    evalType:
                      type: string
                      const: ai
                      description: AI-based evaluation type.
                    guidelines:
                      type: string
                      description: Guidelines for AI evaluation on how to score the metric.
                    promptTemplate:
                      type: string
                      description: >-
                        The complete prompt template for AI evaluation. Should include placeholders for
                        dynamic content.
                    evalModelName:
                      type: string
                      default: gpt-4o
                      description: The AI model to use for evaluation.
                    temperature:
                      type: number
                      minimum: 0
                      maximum: 2
                      default: 0
                      description: The temperature for AI evaluation (0-2).
                    outputType:
                      type: string
                      const: float
                      description: Float output type (0-1).
                    passingThreshold:
                      type: number
                      minimum: 0
                      maximum: 1
                      default: 0.9
                      description: Threshold for determining pass/fail from float scores (0.0-1.0).
                  required:
                    - name
                    - evalType
                    - promptTemplate
                    - outputType
                  description: A Metric with AI evaluation and float output.
                  title: AI float metric
                - type: object
                  properties:
                    name:
                      type: string
                      description: The name of the Metric.
                    description:
                      type:
                        - string
                        - "null"
                      default: null
                      description: The description of the Metric.
                    evalType:
                      type: string
                      const: human
                      description: Human-based evaluation type.
                    guidelines:
                      type: string
                      description: Guidelines for human evaluators.
                    outputType:
                      type: string
                      const: float
                      description: Float output type (0-1).
                    passingThreshold:
                      type: number
                      minimum: 0
                      maximum: 1
                      default: 0.9
                      description: Threshold for determining pass/fail from float scores (0.0-1.0).
                  required:
                    - name
                    - evalType
                    - outputType
                  description: A Metric with human evaluation and float output.
                  title: Human float metric
                - type: object
                  properties:
                    name:
                      type: string
                      description: The name of the Metric.
                    description:
                      type:
                        - string
                        - "null"
                      default: null
                      description: The description of the Metric.
                    evalType:
                      type: string
                      const: heuristic
                      description: Heuristic-based evaluation type.
                    guidelines:
                      type: string
                      description: Guidelines for heuristic evaluation logic.
                    outputType:
                      type: string
                      const: float
                      description: Float output type (0-1).
                    passingThreshold:
                      type: number
                      minimum: 0
                      maximum: 1
                      default: 0.9
                      description: Threshold for determining pass/fail from float scores (0.0-1.0).
                  required:
                    - name
                    - evalType
                    - outputType
                  description: A Metric with heuristic evaluation and float output.
                  title: Heuristic float metric
                - type: object
                  properties:
                    name:
                      type: string
                      description: The name of the Metric.
                    description:
                      type:
                        - string
                        - "null"
                      default: null
                      description: The description of the Metric.
                    evalType:
                      type: string
                      const: ai
                      description: AI-based evaluation type.
                    guidelines:
                      type: string
                      description: Guidelines for AI evaluation on how to score the metric.
                    promptTemplate:
                      type: string
                      description: >-
                        The complete prompt template for AI evaluation. Should include placeholders for
                        dynamic content.
                    evalModelName:
                      type: string
                      default: gpt-4o
                      description: The AI model to use for evaluation.
                    temperature:
                      type: number
                      minimum: 0
                      maximum: 2
                      default: 0
                      description: The temperature for AI evaluation (0-2).
                    outputType:
                      type: string
                      const: boolean
                      description: Boolean output type.
                  required:
                    - name
                    - evalType
                    - promptTemplate
                    - outputType
                  description: A Metric with AI evaluation and boolean output.
                  title: AI boolean metric
                - type: object
                  properties:
                    name:
                      type: string
                      description: The name of the Metric.
                    description:
                      type:
                        - string
                        - "null"
                      default: null
                      description: The description of the Metric.
                    evalType:
                      type: string
                      const: human
                      description: Human-based evaluation type.
                    guidelines:
                      type: string
                      description: Guidelines for human evaluators.
                    outputType:
                      type: string
                      const: boolean
                      description: Boolean output type.
                  required:
                    - name
                    - evalType
                    - outputType
                  description: A Metric with human evaluation and boolean output.
                  title: Human boolean metric
                - type: object
                  properties:
                    name:
                      type: string
                      description: The name of the Metric.
                    description:
                      type:
                        - string
                        - "null"
                      default: null
                      description: The description of the Metric.
                    evalType:
                      type: string
                      const: heuristic
                      description: Heuristic-based evaluation type.
                    guidelines:
                      type: string
                      description: Guidelines for heuristic evaluation logic.
                    outputType:
                      type: string
                      const: boolean
                      description: Boolean output type.
                  required:
                    - name
                    - evalType
                    - outputType
                  description: A Metric with heuristic evaluation and boolean output.
                  title: Heuristic boolean metric
            examples:
              Create AI Metric:
                value:
                  name: Response Accuracy
                  description: Evaluates if the response is factually accurate
                  outputType: boolean
                  evalType: ai
                  guidelines: Check if the response contains factually correct information
                  promptTemplate: "Please evaluate if the following response is factually accurate: {{outputs.response}}"
                  evalModelName: gpt-4o
                  temperature: 0.1
                summary: Created AI Metric
                description: Response after successfully creating an AI metric.
              Create Human Metric:
                value:
                  name: Response Quality
                  description: Human evaluation of response quality
                  outputType: int
                  evalType: human
                  guidelines: Rate the quality of the response on a scale of 1-5
                  passingThreshold: 3
                summary: Create Human Metric
                description: Request to create a human-evaluated metric with integer scoring.
              Create Heuristic Metric:
                value:
                  name: Response Length
                  description: Checks if response meets length requirements
                  outputType: boolean
                  evalType: heuristic
                  guidelines: Response should be between 50-200 characters
                summary: Create Heuristic Metric
                description: Request to create a rule-based metric that uses algorithmic validation.
              Create Simple Heuristic Metric:
                value:
                  name: Non-Empty Response
                  description: Checks if response is not empty
                  guidelines: Checks if response is not empty
                  outputType: boolean
                  evalType: heuristic
                summary: Create Simple Heuristic Metric
                description: Request to create a simple heuristic metric without guidelines.
      responses:
        "201":
          description: Metric created successfully
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/Metric"
              examples:
                Created AI Metric:
                  value:
                    id: "456"
                    name: Response Accuracy
                    description: Evaluates if the response is factually accurate
                    outputType: boolean
                    evalType: ai
                    guidelines: Check if the response contains factually correct information
                    promptTemplate: "Please evaluate if the following response is factually accurate: {{response}}"
                    evalModelName: gpt-4o
                    temperature: 0.1
                  summary: Created AI Metric
                  description: Response after successfully creating an AI metric.
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: |-
            import Scorecard from 'scorecard-ai';

            const client = new Scorecard({
              apiKey: 'My API Key',
            });

            const metric = await client.metrics.create('314', {
              evalType: 'ai',
              name: 'Response Accuracy',
              outputType: 'boolean',
              promptTemplate: 'Please evaluate if the following response is factually accurate: {{outputs.response}}',
              description: 'Evaluates if the response is factually accurate',
              evalModelName: 'gpt-4o',
              guidelines: 'Check if the response contains factually correct information',
              temperature: 0.1,
            });

            console.log(metric);
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            metric = client.metrics.create(
                project_id="314",
                eval_type="ai",
                name="Response Accuracy",
                output_type="boolean",
                prompt_template="Please evaluate if the following response is factually accurate: {{outputs.response}}",
                description="Evaluates if the response is factually accurate",
                eval_model_name="gpt-4o",
                guidelines="Check if the response contains factually correct information",
                temperature=0.1,
            )
            print(metric)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/projects/$PROJECT_ID/metrics \
                -X POST \
                -H "Authorization: Bearer $SCORECARD_API_KEY"
  /metrics/{metricId}:
    get:
      operationId: getMetric
      summary: Get Metric
      description: Retrieve a specific Metric by ID.
      parameters:
        - in: path
          name: metricId
          description: The ID of the Metric to retrieve.
          schema:
            type: string
            example: "321"
          required: true
      responses:
        "200":
          description: Successfully retrieved Metric.
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/Metric"
              examples:
                AI Metric:
                  summary: AI Metric
                  description: Example response showing an AI metric with boolean output.
                  value:
                    id: "654"
                    name: Response Accuracy
                    description: Evaluates factual correctness
                    evalType: ai
                    guidelines: Check if the response is factually correct.
                    promptTemplate: "Evaluate if the following response is factually correct: {{outputs.response}}"
                    evalModelName: gpt-4o
                    temperature: 0.1
                    outputType: boolean
                Human Float Metric:
                  summary: Human Float Metric
                  description: Example response showing a human evaluated metric with float output.
                  value:
                    id: "655"
                    name: Response Quality
                    description: Human review of response quality
                    evalType: human
                    guidelines: Rate the response quality between 0 and 1.
                    outputType: float
                    passingThreshold: 0.85
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: |-
            import Scorecard from 'scorecard-ai';

            const client = new Scorecard({
              apiKey: 'My API Key',
            });

            const metric = await client.metrics.get('321');

            console.log(metric);
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            metric = client.metrics.get(
                "321",
            )
            print(metric)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/metrics/$METRIC_ID \
                -H "Authorization: Bearer $SCORECARD_API_KEY"
    patch:
      operationId: updateMetric
      summary: Update Metric
      description: >-
        Update an existing Metric. You must specify the evalType and outputType of the metric. The structure
        of a metric depends on the evalType and outputType of the metric.
      parameters:
        - in: path
          name: metricId
          description: The ID of the Metric to update.
          schema:
            type: string
            example: "321"
          required: true
      requestBody:
        content:
          application/json:
            schema:
              anyOf:
                - type: object
                  properties:
                    name:
                      type: string
                      description: The name of the Metric.
                    description:
                      type:
                        - string
                        - "null"
                      default: null
                      description: The description of the Metric.
                    evalType:
                      type: string
                      const: ai
                      description: AI-based evaluation type.
                    guidelines:
                      type: string
                      description: Guidelines for AI evaluation on how to score the metric.
                    promptTemplate:
                      type: string
                      description: >-
                        The complete prompt template for AI evaluation. Should include placeholders for
                        dynamic content.
                    evalModelName:
                      type: string
                      default: gpt-4o
                      description: The AI model to use for evaluation.
                    temperature:
                      type: number
                      minimum: 0
                      maximum: 2
                      default: 0
                      description: The temperature for AI evaluation (0-2).
                    outputType:
                      type: string
                      const: int
                      description: Integer output type.
                    passingThreshold:
                      type: integer
                      minimum: 1
                      maximum: 5
                      default: 4
                      description: The threshold for determining pass/fail from integer scores (1-5).
                  required:
                    - evalType
                    - outputType
                  description: A Metric with AI evaluation and integer output.
                  title: AI int metric
                - type: object
                  properties:
                    name:
                      type: string
                      description: The name of the Metric.
                    description:
                      type:
                        - string
                        - "null"
                      default: null
                      description: The description of the Metric.
                    evalType:
                      type: string
                      const: human
                      description: Human-based evaluation type.
                    guidelines:
                      type: string
                      description: Guidelines for human evaluators.
                    outputType:
                      type: string
                      const: int
                      description: Integer output type.
                    passingThreshold:
                      type: integer
                      minimum: 1
                      maximum: 5
                      default: 4
                      description: The threshold for determining pass/fail from integer scores (1-5).
                  required:
                    - evalType
                    - outputType
                  description: A Metric with human evaluation and integer output.
                  title: Human int metric
                - type: object
                  properties:
                    name:
                      type: string
                      description: The name of the Metric.
                    description:
                      type:
                        - string
                        - "null"
                      default: null
                      description: The description of the Metric.
                    evalType:
                      type: string
                      const: heuristic
                      description: Heuristic-based evaluation type.
                    guidelines:
                      type: string
                      description: Guidelines for heuristic evaluation logic.
                    outputType:
                      type: string
                      const: int
                      description: Integer output type.
                    passingThreshold:
                      type: integer
                      minimum: 1
                      maximum: 5
                      default: 4
                      description: The threshold for determining pass/fail from integer scores (1-5).
                  required:
                    - evalType
                    - outputType
                  description: A Metric with heuristic evaluation and integer output.
                  title: Heuristic int metric
                - type: object
                  properties:
                    name:
                      type: string
                      description: The name of the Metric.
                    description:
                      type:
                        - string
                        - "null"
                      default: null
                      description: The description of the Metric.
                    evalType:
                      type: string
                      const: ai
                      description: AI-based evaluation type.
                    guidelines:
                      type: string
                      description: Guidelines for AI evaluation on how to score the metric.
                    promptTemplate:
                      type: string
                      description: >-
                        The complete prompt template for AI evaluation. Should include placeholders for
                        dynamic content.
                    evalModelName:
                      type: string
                      default: gpt-4o
                      description: The AI model to use for evaluation.
                    temperature:
                      type: number
                      minimum: 0
                      maximum: 2
                      default: 0
                      description: The temperature for AI evaluation (0-2).
                    outputType:
                      type: string
                      const: float
                      description: Float output type (0-1).
                    passingThreshold:
                      type: number
                      minimum: 0
                      maximum: 1
                      default: 0.9
                      description: Threshold for determining pass/fail from float scores (0.0-1.0).
                  required:
                    - evalType
                    - outputType
                  description: A Metric with AI evaluation and float output.
                  title: AI float metric
                - type: object
                  properties:
                    name:
                      type: string
                      description: The name of the Metric.
                    description:
                      type:
                        - string
                        - "null"
                      default: null
                      description: The description of the Metric.
                    evalType:
                      type: string
                      const: human
                      description: Human-based evaluation type.
                    guidelines:
                      type: string
                      description: Guidelines for human evaluators.
                    outputType:
                      type: string
                      const: float
                      description: Float output type (0-1).
                    passingThreshold:
                      type: number
                      minimum: 0
                      maximum: 1
                      default: 0.9
                      description: Threshold for determining pass/fail from float scores (0.0-1.0).
                  required:
                    - evalType
                    - outputType
                  description: A Metric with human evaluation and float output.
                  title: Human float metric
                - type: object
                  properties:
                    name:
                      type: string
                      description: The name of the Metric.
                    description:
                      type:
                        - string
                        - "null"
                      default: null
                      description: The description of the Metric.
                    evalType:
                      type: string
                      const: heuristic
                      description: Heuristic-based evaluation type.
                    guidelines:
                      type: string
                      description: Guidelines for heuristic evaluation logic.
                    outputType:
                      type: string
                      const: float
                      description: Float output type (0-1).
                    passingThreshold:
                      type: number
                      minimum: 0
                      maximum: 1
                      default: 0.9
                      description: Threshold for determining pass/fail from float scores (0.0-1.0).
                  required:
                    - evalType
                    - outputType
                  description: A Metric with heuristic evaluation and float output.
                  title: Heuristic float metric
                - type: object
                  properties:
                    name:
                      type: string
                      description: The name of the Metric.
                    description:
                      type:
                        - string
                        - "null"
                      default: null
                      description: The description of the Metric.
                    evalType:
                      type: string
                      const: ai
                      description: AI-based evaluation type.
                    guidelines:
                      type: string
                      description: Guidelines for AI evaluation on how to score the metric.
                    promptTemplate:
                      type: string
                      description: >-
                        The complete prompt template for AI evaluation. Should include placeholders for
                        dynamic content.
                    evalModelName:
                      type: string
                      default: gpt-4o
                      description: The AI model to use for evaluation.
                    temperature:
                      type: number
                      minimum: 0
                      maximum: 2
                      default: 0
                      description: The temperature for AI evaluation (0-2).
                    outputType:
                      type: string
                      const: boolean
                      description: Boolean output type.
                  required:
                    - evalType
                    - outputType
                  description: A Metric with AI evaluation and boolean output.
                  title: AI boolean metric
                - type: object
                  properties:
                    name:
                      type: string
                      description: The name of the Metric.
                    description:
                      type:
                        - string
                        - "null"
                      default: null
                      description: The description of the Metric.
                    evalType:
                      type: string
                      const: human
                      description: Human-based evaluation type.
                    guidelines:
                      type: string
                      description: Guidelines for human evaluators.
                    outputType:
                      type: string
                      const: boolean
                      description: Boolean output type.
                  required:
                    - evalType
                    - outputType
                  description: A Metric with human evaluation and boolean output.
                  title: Human boolean metric
                - type: object
                  properties:
                    name:
                      type: string
                      description: The name of the Metric.
                    description:
                      type:
                        - string
                        - "null"
                      default: null
                      description: The description of the Metric.
                    evalType:
                      type: string
                      const: heuristic
                      description: Heuristic-based evaluation type.
                    guidelines:
                      type: string
                      description: Guidelines for heuristic evaluation logic.
                    outputType:
                      type: string
                      const: boolean
                      description: Boolean output type.
                  required:
                    - evalType
                    - outputType
                  description: A Metric with heuristic evaluation and boolean output.
                  title: Heuristic boolean metric
            examples:
              Update AI metric's prompt template:
                value:
                  evalType: ai
                  outputType: boolean
                  promptTemplate: |-
                    Using the following guidelines, evaluate the response: {{ guidelines }}

                    Response: {{ outputs.response }}

                    Ideal answer: {{ expected.idealResponse }}
                summary: Update AI metric's prompt template
                description: Update the prompt template of an AI boolean metric.
              Update metric name:
                value:
                  evalType: ai
                  outputType: boolean
                  name: Updated Metric Name
                summary: Update metric name
                description: Update the name of an AI boolean metric.
              Update metric output type:
                value:
                  evalType: human
                  outputType: int
                  passingThreshold: 4
                summary: Update metric output type
                description: Update a metric to be a human int evaluated metric.
      responses:
        "200":
          description: Metric updated successfully
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/Metric"
              examples:
                Updated AI metric's prompt template:
                  value:
                    id: "321"
                    name: Response Accuracy
                    description: Evaluates if the response is factually accurate
                    outputType: boolean
                    evalType: ai
                    evalModelName: gpt-4o
                    guidelines: Check if the response contains factually correct information
                    promptTemplate: |-
                      Using the following guidelines, evaluate the response: {{ guidelines }}

                      Response: {{ outputs.response }}

                      Ideal answer: {{ expected.idealResponse }}
                    temperature: 0.1
                  summary: Updated AI metric's prompt template
                  description: Response after successfully updating an AI metric's prompt template.
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: |-
            import Scorecard from 'scorecard-ai';

            const client = new Scorecard({
              apiKey: 'My API Key',
            });

            const metric = await client.metrics.update('321', {
              evalType: 'ai',
              outputType: 'boolean',
              promptTemplate:
                'Using the following guidelines, evaluate the response: {{ guidelines }}\n\nResponse: {{ outputs.response }}\n\nIdeal answer: {{ expected.idealResponse }}',
            });

            console.log(metric);
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            metric = client.metrics.update(
                metric_id="321",
                eval_type="ai",
                output_type="boolean",
                prompt_template="Using the following guidelines, evaluate the response: {{ guidelines }}\n\nResponse: {{ outputs.response }}\n\nIdeal answer: {{ expected.idealResponse }}",
            )
            print(metric)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/metrics/$METRIC_ID \
                -X PATCH \
                -H "Authorization: Bearer $SCORECARD_API_KEY"
    delete:
      operationId: deleteMetric
      summary: Delete Metric
      description: Delete a specific Metric by ID. The metric will be removed from metric groups and monitors.
      parameters:
        - in: path
          name: metricId
          description: The ID of the Metric to delete.
          schema:
            type: string
            example: "321"
          required: true
      responses:
        "200":
          description: Metric deleted successfully
          content:
            application/json:
              schema:
                type: object
                properties:
                  success:
                    type: boolean
                    description: Whether the deletion was successful.
                required:
                  - success
              examples:
                Successful deletion:
                  value:
                    success: true
                  summary: Successful deletion
                  description: Response after successfully deleting a metric.
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: |-
            import Scorecard from 'scorecard-ai';

            const client = new Scorecard({
              apiKey: 'My API Key',
            });

            const metric = await client.metrics.delete('321');

            console.log(metric.success);
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            metric = client.metrics.delete(
                "321",
            )
            print(metric.success)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/metrics/$METRIC_ID \
                -X DELETE \
                -H "Authorization: Bearer $SCORECARD_API_KEY"
  /projects/{projectId}/runs:
    get:
      operationId: listRuns
      summary: List Runs
      description: >-
        Retrieve a paginated list of all Runs for a Project. Runs are ordered by creation date, most recent
        first.
      parameters:
        - in: path
          name: projectId
          description: The ID of the Run's Project.
          schema:
            type: string
            example: "314"
          required: true
        - in: query
          name: limit
          description: Maximum number of items to return (1-100). Use with `cursor` for pagination through large sets.
          schema:
            type: integer
            exclusiveMinimum: 0
            default: 20
            example: 20
        - in: query
          name: cursor
          description: >-
            Cursor for pagination. Pass the `nextCursor` from the previous response to get the next page of
            results.
          schema:
            type: string
            example: "123"
      responses:
        "200":
          description: Successfully retrieved list of runs.
          content:
            application/json:
              schema:
                type: object
                properties:
                  data:
                    type: array
                    items:
                      $ref: "#/components/schemas/Run"
                  nextCursor:
                    type:
                      - string
                      - "null"
                  hasMore:
                    type: boolean
                  total:
                    type: integer
                    minimum: 0
                required:
                  - data
                  - nextCursor
                  - hasMore
              examples:
                Run list with pagination:
                  value:
                    data:
                      - id: "135"
                        testsetId: "246"
                        systemId: 3fa85f64-5717-4562-b3fc-2c963f66afa6
                        systemVersionId: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
                        metricIds:
                          - "789"
                          - "101"
                        metricVersionIds:
                          - 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
                          - 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf1
                        numRecords: 10
                        numExpectedRecords: 10
                        numScores: 20
                        status: completed
                      - id: "136"
                        testsetId: null
                        systemId: null
                        systemVersionId: null
                        metricIds:
                          - "789"
                        metricVersionIds:
                          - 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
                        numRecords: 5
                        numExpectedRecords: null
                        numScores: 3
                        status: running_scoring
                    nextCursor: 8fb15f74-2918-4982-a4fc-9c157f77dca8
                    hasMore: true
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: |-
            import Scorecard from 'scorecard-ai';

            const client = new Scorecard({
              apiKey: 'My API Key',
            });

            // Automatically fetches more pages as needed.
            for await (const run of client.runs.list('314')) {
              console.log(run.id);
            }
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            page = client.runs.list(
                project_id="314",
            )
            page = page.data[0]
            print(page.id)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/projects/$PROJECT_ID/runs \
                -H "Authorization: Bearer $SCORECARD_API_KEY"
    post:
      operationId: createRun
      summary: Create Run
      description: Create a new Run.
      parameters:
        - in: path
          name: projectId
          description: The ID of the Project.
          schema:
            type: string
            example: "314"
          required: true
      requestBody:
        content:
          application/json:
            schema:
              type: object
              properties:
                testsetId:
                  type:
                    - string
                    - "null"
                  default: null
                  description: The ID of the Testset this Run is testing.
                systemVersionId:
                  type:
                    - string
                    - "null"
                  format: uuid
                  description: The ID of the system version this Run is using.
                metricIds:
                  type: array
                  items:
                    type: string
                  description: The IDs of the metrics this Run is using.
              required:
                - metricIds
            examples:
              Create Run with a Testset:
                value:
                  testsetId: "246"
                  systemVersionId: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
                  metricIds:
                    - "789"
                    - "101"
                summary: Create Run with a Testset
                description: Request to create a Run with a Testset and metrics.
              Create Run without a Testset:
                value:
                  systemVersionId: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
                  metricIds:
                    - "789"
                summary: Create Run without a Testset
                description: Request to create a Run with metrics but no Testset.
      responses:
        "201":
          description: Run created successfully
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/Run"
              examples:
                Created Run with Testset:
                  value:
                    id: "135"
                    testsetId: "246"
                    systemId: 3fa85f64-5717-4562-b3fc-2c963f66afa6
                    systemVersionId: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
                    metricIds:
                      - "789"
                      - "101"
                    metricVersionIds:
                      - 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
                      - 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf1
                    numRecords: 0
                    numExpectedRecords: 10
                    numScores: 0
                    status: awaiting_execution
                  summary: Created Run with Testset
                  description: Response after successfully creating a Run with a Testset.
                Created Run without Testset:
                  value:
                    id: "135"
                    systemId: 3fa85f64-5717-4562-b3fc-2c963f66afa6
                    systemVersionId: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
                    metricIds:
                      - "789"
                    metricVersionIds:
                      - 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
                    numRecords: 0
                    numExpectedRecords: 0
                    numScores: 0
                    status: awaiting_execution
                  summary: Created Run without Testset
                  description: Response after successfully creating a Run without a Testset.
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: |-
            import Scorecard from 'scorecard-ai';

            const client = new Scorecard({
              apiKey: 'My API Key',
            });

            const run = await client.runs.create('314', {
              metricIds: ['789', '101'],
              systemVersionId: '87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0',
              testsetId: '246',
            });

            console.log(run.id);
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            run = client.runs.create(
                project_id="314",
                metric_ids=["789", "101"],
                system_version_id="87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0",
                testset_id="246",
            )
            print(run.id)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/projects/$PROJECT_ID/runs \
                -H 'Content-Type: application/json' \
                -H "Authorization: Bearer $SCORECARD_API_KEY" \
                -d '{
                      "metricIds": [
                        "789",
                        "101"
                      ]
                    }'
  /runs/{runId}:
    get:
      operationId: getRun
      summary: Get Run
      description: Retrieve a specific Run by ID.
      parameters:
        - in: path
          name: runId
          description: The ID of the Run to retrieve.
          schema:
            type: string
            example: "135"
          required: true
      responses:
        "200":
          description: Successfully retrieved Run.
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/Run"
              examples:
                Run with Testset:
                  value:
                    id: "135"
                    testsetId: "246"
                    systemId: 3fa85f64-5717-4562-b3fc-2c963f66afa6
                    systemVersionId: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
                    metricIds:
                      - "789"
                      - "101"
                    metricVersionIds:
                      - 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
                      - 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf1
                    numRecords: 10
                    numExpectedRecords: 10
                    numScores: 20
                    status: completed
                  summary: Run with Testset
                  description: Example response showing a completed Run with a Testset.
                Run without Testset:
                  value:
                    id: "136"
                    testsetId: null
                    systemId: null
                    systemVersionId: null
                    metricIds:
                      - "789"
                    metricVersionIds:
                      - 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
                    numRecords: 5
                    numExpectedRecords: null
                    numScores: 3
                    status: running_scoring
                  summary: Run without Testset
                  description: Example response showing a Run without a Testset that's currently running scoring.
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: |-
            import Scorecard from 'scorecard-ai';

            const client = new Scorecard({
              apiKey: 'My API Key',
            });

            const run = await client.runs.get('135');

            console.log(run.id);
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            run = client.runs.get(
                "135",
            )
            print(run.id)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/runs/$RUN_ID \
                -H "Authorization: Bearer $SCORECARD_API_KEY"
  /runs/{runId}/records:
    post:
      operationId: createRecord
      summary: Create Record
      description: Create a new Record in a Run.
      parameters:
        - in: path
          name: runId
          description: The ID of the Run.
          schema:
            type: string
            example: "135"
          required: true
      requestBody:
        content:
          application/json:
            schema:
              type: object
              properties:
                testcaseId:
                  type: string
                  description: The ID of the Testcase.
                inputs:
                  type: object
                  additionalProperties: true
                  description: The actual inputs sent to the system, which should match the system's input schema.
                expected:
                  type: object
                  additionalProperties: true
                  description: The expected outputs for the Testcase.
                outputs:
                  type: object
                  additionalProperties: true
                  description: The actual outputs from the system.
              required:
                - inputs
                - expected
                - outputs
            examples:
              Create Record:
                value:
                  testcaseId: "248"
                  inputs:
                    question: What is the capital of France?
                  expected:
                    idealAnswer: Paris is the capital of France
                  outputs:
                    response: The capital of France is Paris.
                summary: Create Record
                description: Request to create a Record.
      responses:
        "201":
          description: Record created successfully
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/Record"
              examples:
                Created Record:
                  value:
                    id: "864"
                    runId: "135"
                    testcaseId: "248"
                    inputs:
                      question: What is the capital of France?
                    expected:
                      idealAnswer: Paris is the capital of France
                    outputs:
                      response: The capital of France is Paris.
                  summary: Created Record
                  description: Response after successfully creating a Record.
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: |-
            import Scorecard from 'scorecard-ai';

            const client = new Scorecard({
              apiKey: 'My API Key',
            });

            const record = await client.records.create('135', {
              expected: { idealAnswer: 'Paris is the capital of France' },
              inputs: { question: 'What is the capital of France?' },
              outputs: { response: 'The capital of France is Paris.' },
              testcaseId: '248',
            });

            console.log(record.id);
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            record = client.records.create(
                run_id="135",
                expected={
                    "idealAnswer": "Paris is the capital of France"
                },
                inputs={
                    "question": "What is the capital of France?"
                },
                outputs={
                    "response": "The capital of France is Paris."
                },
                testcase_id="248",
            )
            print(record.id)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/runs/$RUN_ID/records \
                -H 'Content-Type: application/json' \
                -H "Authorization: Bearer $SCORECARD_API_KEY" \
                -d '{
                      "expected": {
                        "idealAnswer": "bar"
                      },
                      "inputs": {
                        "question": "bar"
                      },
                      "outputs": {
                        "response": "bar"
                      }
                    }'
    get:
      operationId: listRecords
      summary: List Records
      description: Retrieve a paginated list of Records for a Run, including all scores for each record.
      parameters:
        - in: path
          name: runId
          description: The ID of the Run to list records for.
          schema:
            type: string
            example: "135"
          required: true
        - in: query
          name: limit
          description: Maximum number of items to return (1-100). Use with `cursor` for pagination through large sets.
          schema:
            type: integer
            exclusiveMinimum: 0
            default: 20
            example: 20
        - in: query
          name: cursor
          description: >-
            Cursor for pagination. Pass the `nextCursor` from the previous response to get the next page of
            results.
          schema:
            type: string
            example: "123"
      responses:
        "200":
          description: Successfully retrieved list of Records with scores.
          content:
            application/json:
              schema:
                type: object
                properties:
                  data:
                    type: array
                    items:
                      $ref: "#/components/schemas/RecordWithScores"
                  nextCursor:
                    type:
                      - string
                      - "null"
                  hasMore:
                    type: boolean
                  total:
                    type: integer
                    minimum: 0
                required:
                  - data
                  - nextCursor
                  - hasMore
              examples:
                Records with scores:
                  value:
                    data:
                      - id: "456"
                        runId: "135"
                        testcaseId: "248"
                        inputs:
                          question: What is the capital of France?
                        expected:
                          idealAnswer: Paris is the capital of France
                        outputs:
                          response: The capital of France is Paris.
                        scores:
                          - recordId: "456"
                            metricConfigId: a1b2c3d4-e5f6-7890-1234-567890abcdef
                            score:
                              binaryScore: true
                              reasoning: The response correctly identifies Paris as the capital.
                          - recordId: "456"
                            metricConfigId: b2c3d4e5-f6a7-8901-2345-67890abcdef0
                            score:
                              intScore: 4
                              reasoning: Good answer but could be more detailed.
                      - id: "457"
                        runId: "135"
                        testcaseId: "249"
                        inputs:
                          question: What is the largest planet in our solar system?
                        expected:
                          idealAnswer: Jupiter
                        outputs:
                          response: Jupiter is the largest planet in our solar system.
                        scores:
                          - recordId: "457"
                            metricConfigId: a1b2c3d4-e5f6-7890-1234-567890abcdef
                            score:
                              binaryScore: true
                              reasoning: Correctly identified Jupiter.
                          - recordId: "457"
                            metricConfigId: b2c3d4e5-f6a7-8901-2345-67890abcdef0
                            score:
                              intScore: 5
                              reasoning: Perfect answer with good detail.
                    nextCursor: "458"
                    hasMore: true
                  summary: Records with scores
                  description: >-
                    Example response showing a paginated list of Records with their associated scores and
                    reasoning.
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: |-
            import Scorecard from 'scorecard-ai';

            const client = new Scorecard({
              apiKey: 'My API Key',
            });

            // Automatically fetches more pages as needed.
            for await (const recordListResponse of client.records.list('135')) {
              console.log(recordListResponse);
            }
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            page = client.records.list(
                run_id="135",
            )
            page = page.data[0]
            print(page)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/runs/$RUN_ID/records \
                -H "Authorization: Bearer $SCORECARD_API_KEY"
  /records/{recordId}:
    delete:
      operationId: deleteRecord
      summary: Delete Record
      description: Delete a specific Record by ID.
      parameters:
        - in: path
          name: recordId
          description: The ID of the Record to delete.
          schema:
            type: string
            example: "777"
          required: true
      responses:
        "200":
          description: Record deleted successfully
          content:
            application/json:
              schema:
                type: object
                properties:
                  success:
                    type: boolean
                    description: Whether the deletion was successful.
                required:
                  - success
              examples:
                Delete Record response:
                  value:
                    success: true
                  summary: Delete Record success
                  description: Response indicating successful deletion of the Record.
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: |-
            import Scorecard from 'scorecard-ai';

            const client = new Scorecard({
              apiKey: 'My API Key',
            });

            const record = await client.records.delete('777');

            console.log(record.success);
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            record = client.records.delete(
                "777",
            )
            print(record.success)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/records/$RECORD_ID \
                -X DELETE \
                -H "Authorization: Bearer $SCORECARD_API_KEY"
  /records/{recordId}/annotations:
    get:
      operationId: listAnnotations
      summary: List Annotations
      description: List all annotations (ratings and comments) for a specific Record.
      parameters:
        - in: path
          name: recordId
          description: The ID of the Record to list annotations for.
          schema:
            type: string
            example: "777"
          required: true
      responses:
        "200":
          description: Successfully retrieved list of Annotations for the Record.
          content:
            application/json:
              schema:
                type: object
                properties:
                  data:
                    type: array
                    items:
                      $ref: "#/components/schemas/Annotation"
                required:
                  - data
              examples:
                Annotations for a record:
                  value:
                    data:
                      - id: "1"
                        recordId: "777"
                        spanId: null
                        rating: true
                        comment: Great response!
                        userId: user_abc123
                        createdAt: "2026-03-01T12:00:00.000Z"
                      - id: "2"
                        recordId: "777"
                        spanId: span_xyz
                        rating: false
                        comment: This answer was incorrect for the given context.
                        userId: user_def456
                        createdAt: "2026-03-02T15:30:00.000Z"
                  summary: Annotations for a record
                  description: Example response showing annotations with ratings and comments on a Record.
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: |-
            import Scorecard from 'scorecard-ai';

            const client = new Scorecard({
              apiKey: 'My API Key',
            });

            const annotations = await client.records.annotations.list('777');

            console.log(annotations.data);
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            annotations = client.records.annotations.list(
                "777",
            )
            print(annotations.data)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/records/$RECORD_ID/annotations \
                -H "Authorization: Bearer $SCORECARD_API_KEY"
  /projects/{projectId}/systems:
    get:
      operationId: listSystems
      summary: List systems
      description: Retrieve a paginated list of all systems. Systems are ordered by creation date.
      parameters:
        - in: path
          name: projectId
          description: The ID of the system's Project.
          schema:
            type: string
            example: "314"
          required: true
        - in: query
          name: limit
          description: Maximum number of items to return (1-100). Use with `cursor` for pagination through large sets.
          schema:
            type: integer
            exclusiveMinimum: 0
            default: 20
            example: 20
        - in: query
          name: cursor
          description: >-
            Cursor for pagination. Pass the `nextCursor` from the previous response to get the next page of
            results.
          schema:
            type: string
            example: eyJvZmZzZXQiOjAsInBhZ2VJZCI6ImNvZGUifQ
      responses:
        "200":
          description: Successfully retrieved list of systems.
          content:
            application/json:
              schema:
                type: object
                properties:
                  data:
                    type: array
                    items:
                      $ref: "#/components/schemas/System"
                  nextCursor:
                    type:
                      - string
                      - "null"
                  hasMore:
                    type: boolean
                  total:
                    type: integer
                    minimum: 0
                required:
                  - data
                  - nextCursor
                  - hasMore
              examples:
                System list with pagination:
                  value:
                    data:
                      - id: 3fa85f64-5717-4562-b3fc-2c963f66afa6
                        name: GPT-4 Chatbot
                        description: Production chatbot powered by GPT-4
                        productionVersion:
                          id: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
                          systemId: 3fa85f64-5717-4562-b3fc-2c963f66afa6
                          name: Version 2 (Low Temperature)
                          config:
                            temperature: 0.1
                            maxTokens: 1024
                        versions:
                          - id: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
                            name: Version 2 (Low Temperature)
                          - id: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf1
                            name: Version 1 (High Temperature)
                      - id: 8fb15f74-2918-4982-a4fc-9c157f77dca7
                        name: RAG System
                        description: Retrieval-augmented generation system for company knowledge base
                        productionVersion:
                          id: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf4
                          systemId: 8fb15f74-2918-4982-a4fc-9c157f77dca7
                          name: Best version
                          config:
                            top_k: 2
                            chunk_size: 512
                        versions:
                          - id: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf4
                            name: Best version
                    nextCursor: 8fb15f74-2918-4982-a4fc-9c157f77dca8
                    hasMore: true
                  summary: System list with pagination
                  description: Example response showing a paginated list of two systems.
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: |-
            import Scorecard from 'scorecard-ai';

            const client = new Scorecard({
              apiKey: 'My API Key',
            });

            // Automatically fetches more pages as needed.
            for await (const system of client.systems.list('314')) {
              console.log(system.id);
            }
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            page = client.systems.list(
                project_id="314",
            )
            page = page.data[0]
            print(page.id)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/projects/$PROJECT_ID/systems \
                -H "Authorization: Bearer $SCORECARD_API_KEY"
    post:
      operationId: upsertSystem
      summary: Create (upsert) system
      description: Create a new system. If one with the same name in the project exists, it updates it instead.
      parameters:
        - in: path
          name: projectId
          description: The ID of the system's Project.
          schema:
            type: string
            example: "314"
          required: true
      requestBody:
        content:
          application/json:
            schema:
              type: object
              properties:
                description:
                  type: string
                  default: ""
                  description: The description of the system.
                name:
                  type: string
                  default: Default system
                  description: The name of the system. Should be unique within the project. Default is "Default system"
                config:
                  type: object
                  additionalProperties: true
                  description: The configuration of the system.
              required:
                - config
            examples:
              Create chatbot system:
                value:
                  name: GPT-4 Chatbot
                  description: Production chatbot powered by GPT-4
                  config:
                    temperature: 0.1
                    maxTokens: 1024
                summary: Create chatbot system
                description: Create a system for a GPT-4 powered chatbot.
              Create RAG system:
                value:
                  name: RAG System
                  description: Retrieval-augmented generation system for company knowledge base
                  config:
                    top_k: 5
                    chunk_size: 1024
                summary: Create RAG system
                description: Create a system for a retrieval-augmented generation system.
      responses:
        "201":
          description: System created successfully
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/System"
              examples:
                Created system response:
                  value:
                    id: 3fa85f64-5717-4562-b3fc-2c963f66afa6
                    name: GPT-4 Chatbot
                    description: Production chatbot powered by GPT-4
                    productionVersion:
                      id: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
                      systemId: 3fa85f64-5717-4562-b3fc-2c963f66afa6
                      name: Version 1
                      config:
                        temperature: 0.1
                        maxTokens: 1024
                    versions:
                      - id: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
                        name: Version 1
                  summary: Created system response
                  description: Response after successfully creating a system for a GPT-4 chatbot.
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: |-
            import Scorecard from 'scorecard-ai';

            const client = new Scorecard({
              apiKey: 'My API Key',
            });

            const system = await client.systems.upsert('314', {
              config: { temperature: 0.1, maxTokens: 1024 },
              description: 'Production chatbot powered by GPT-4',
              name: 'GPT-4 Chatbot',
            });

            console.log(system.id);
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            system = client.systems.upsert(
                project_id="314",
                config={
                    "temperature": 0.1,
                    "maxTokens": 1024,
                },
                description="Production chatbot powered by GPT-4",
                name="GPT-4 Chatbot",
            )
            print(system.id)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/projects/$PROJECT_ID/systems \
                -H 'Content-Type: application/json' \
                -H "Authorization: Bearer $SCORECARD_API_KEY" \
                -d '{
                      "config": {
                        "temperature": "bar",
                        "maxTokens": "bar"
                      }
                    }'
  /systems/{systemId}:
    get:
      operationId: getSystem
      summary: Get system
      description: Retrieve a specific system by ID.
      parameters:
        - in: path
          name: systemId
          description: The ID of the system to retrieve.
          schema:
            type: string
            format: uuid
            example: 12345678-0a8b-4f66-b6f3-2ddcfa097257
          required: true
      responses:
        "200":
          description: Successfully retrieved system.
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/System"
              examples:
                Complete system details:
                  value:
                    id: 12345678-0a8b-4f66-b6f3-2ddcfa097257
                    name: GPT-4 Chatbot
                    description: Production chatbot powered by GPT-4
                    productionVersion:
                      id: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
                      systemId: 3fa85f64-5717-4562-b3fc-2c963f66afa6
                      name: Version 2 (Low Temperature)
                      config:
                        temperature: 0.1
                        maxTokens: 1024
                    versions:
                      - id: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
                        name: Version 2 (Low Temperature)
                      - id: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf1
                        name: Version 1 (High Temperature)
                  summary: Complete system details
                  description: Example response showing all details of a system with two versions.
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: |-
            import Scorecard from 'scorecard-ai';

            const client = new Scorecard({
              apiKey: 'My API Key',
            });

            const system = await client.systems.get('12345678-0a8b-4f66-b6f3-2ddcfa097257');

            console.log(system.id);
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            system = client.systems.get(
                "12345678-0a8b-4f66-b6f3-2ddcfa097257",
            )
            print(system.id)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/systems/$SYSTEM_ID \
                -H "Authorization: Bearer $SCORECARD_API_KEY"
    patch:
      operationId: updateSystem
      summary: Update system
      description: |-
        Update an existing system. Only the fields provided in the request body will be updated.
        If a field is provided, the new content will replace the existing content.
        If a field is not provided, the existing content will remain unchanged.
      parameters:
        - in: path
          name: systemId
          description: The ID of the system to update.
          schema:
            type: string
            format: uuid
            example: 12345678-0a8b-4f66-b6f3-2ddcfa097257
          required: true
      requestBody:
        content:
          application/json:
            schema:
              type: object
              properties:
                name:
                  type: string
                  description: The name of the system. Unique within the project.
                description:
                  type: string
                  default: ""
                  description: The description of the system.
                productionVersionId:
                  type: string
                  format: uuid
                  description: The ID of the production version of the system.
            examples:
              Update system production version:
                value:
                  productionVersionId: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf3
                summary: Update production config
                description: Updates a system to mark an existing system version as the production version.
              Update system name:
                value:
                  name: GPT-4 Turbo Chatbot
                  description: Updated production chatbot powered by GPT-4 Turbo
                summary: Update system metadata
                description: >-
                  Simple metadata update without changing any system versions. Updates only the name and
                  description fields while preserving the system config.
      responses:
        "200":
          description: System updated successfully
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/System"
              examples:
                Updated system production version:
                  value:
                    id: 12345678-0a8b-4f66-b6f3-2ddcfa097257
                    name: GPT-4 Turbo Chatbot
                    description: Updated production chatbot powered by GPT-4 Turbo
                    productionVersion:
                      id: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf3
                      systemId: 12345678-0a8b-4f66-b6f3-2ddcfa097257
                      name: Version 3 (with system prompt)
                      config:
                        temperature: 0.5
                        maxTokens: 1024
                        systemPrompt: You are an extremely helpful assistant.
                    versions:
                      - id: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf3
                        name: Version 3 (with system prompt)
                      - id: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
                        name: Version 2 (Low Temperature)
                      - id: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf1
                        name: Version 1 (High Temperature)
                  summary: Updated system
                  description: Result after updating the production config of a system.
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: |-
            import Scorecard from 'scorecard-ai';

            const client = new Scorecard({
              apiKey: 'My API Key',
            });

            const system = await client.systems.update('12345678-0a8b-4f66-b6f3-2ddcfa097257', {
              productionVersionId: '87654321-4d3b-4ae4-8c7a-4b6e2a19ccf3',
            });

            console.log(system.id);
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            system = client.systems.update(
                system_id="12345678-0a8b-4f66-b6f3-2ddcfa097257",
                production_version_id="87654321-4d3b-4ae4-8c7a-4b6e2a19ccf3",
            )
            print(system.id)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/systems/$SYSTEM_ID \
                -X PATCH \
                -H "Authorization: Bearer $SCORECARD_API_KEY"
    delete:
      operationId: deleteSystem
      summary: Delete system
      description: Delete a system definition by ID. This will not delete associated system versions.
      parameters:
        - in: path
          name: systemId
          description: The ID of the system to delete.
          schema:
            type: string
            format: uuid
            example: 12345678-0a8b-4f66-b6f3-2ddcfa097257
          required: true
      responses:
        "200":
          description: System deleted successfully
          content:
            application/json:
              schema:
                type: object
                properties:
                  success:
                    type: boolean
                    description: Whether the deletion was successful.
                required:
                  - success
              examples:
                Delete system response:
                  value:
                    success: true
                  summary: Delete system success
                  description: Response indicating successful deletion of the system.
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: |-
            import Scorecard from 'scorecard-ai';

            const client = new Scorecard({
              apiKey: 'My API Key',
            });

            const system = await client.systems.delete('12345678-0a8b-4f66-b6f3-2ddcfa097257');

            console.log(system.success);
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            system = client.systems.delete(
                "12345678-0a8b-4f66-b6f3-2ddcfa097257",
            )
            print(system.success)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/systems/$SYSTEM_ID \
                -X DELETE \
                -H "Authorization: Bearer $SCORECARD_API_KEY"
  /systems/versions/{systemVersionId}:
    get:
      operationId: getSystemVersion
      summary: Get system version
      description: Retrieve a specific system version by ID.
      parameters:
        - in: path
          name: systemVersionId
          description: The ID of the system version to retrieve.
          schema:
            type: string
            format: uuid
            example: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
          required: true
      responses:
        "200":
          description: Successfully retrieved system version.
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/SystemVersion"
              examples:
                System version details:
                  value:
                    id: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
                    systemId: 12345678-0a8b-4f66-b6f3-2ddcfa097257
                    name: Production (Low Temperature)
                    config:
                      temperature: 0.1
                      maxTokens: 1024
                      model: gpt-4-turbo
                  summary: System version details
                  description: Example response showing the complete details of a valid system version.
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: |-
            import Scorecard from 'scorecard-ai';

            const client = new Scorecard({
              apiKey: 'My API Key',
            });

            const systemVersion = await client.systems.versions.get('87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0');

            console.log(systemVersion.id);
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            system_version = client.systems.versions.get(
                "87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0",
            )
            print(system_version.id)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/systems/versions/$SYSTEM_VERSION_ID \
                -H "Authorization: Bearer $SCORECARD_API_KEY"
  /systems/{systemId}/versions:
    post:
      operationId: upsertSystemVersion
      summary: Upsert system version
      description: >-
        Create a new system version if it does not already exist. Does **not** set the created version to be
        the system's production version.


        If there is already a system version with the same config, its name will be updated.
      parameters:
        - in: path
          name: systemId
          description: The ID of the system to create the system version for.
          schema:
            type: string
            format: uuid
            example: 12345678-0a8b-4f66-b6f3-2ddcfa097257
          required: true
      requestBody:
        content:
          application/json:
            schema:
              type: object
              properties:
                config:
                  type: object
                  additionalProperties: true
                  description: The configuration of the system version.
                name:
                  type: string
                  description: >-
                    The name of the system version. If creating a new system version and the name isn't
                    provided, it will be autogenerated.
              required:
                - config
            examples:
              Create system version:
                value:
                  name: "Test model: Gemini"
                  config:
                    temperature: 0.5
                    maxTokens: 1024
                    model: gemini-2.0-flash
                summary: Create system version
                description: Create a new system version.
      responses:
        "200":
          description: Successfully upserted system version.
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/SystemVersion"
              examples:
                System version details:
                  value:
                    id: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
                    systemId: 12345678-0a8b-4f66-b6f3-2ddcfa097257
                    name: Production (Low Temperature)
                    config:
                      temperature: 0.1
                      maxTokens: 1024
                      model: gpt-4-turbo
                  summary: System version details
                  description: Example response showing the complete details of a valid system version.
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: >-
            import Scorecard from 'scorecard-ai';


            const client = new Scorecard({
              apiKey: 'My API Key',
            });


            const systemVersion = await client.systems.versions.upsert('12345678-0a8b-4f66-b6f3-2ddcfa097257',
            {
              config: { temperature: 0.5, maxTokens: 1024, model: 'gemini-2.0-flash' },
              name: 'Test model: Gemini',
            });


            console.log(systemVersion.id);
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            system_version = client.systems.versions.upsert(
                system_id="12345678-0a8b-4f66-b6f3-2ddcfa097257",
                config={
                    "temperature": 0.5,
                    "maxTokens": 1024,
                    "model": "gemini-2.0-flash",
                },
                name="Test model: Gemini",
            )
            print(system_version.id)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/systems/$SYSTEM_ID/versions \
                -H 'Content-Type: application/json' \
                -H "Authorization: Bearer $SCORECARD_API_KEY" \
                -d '{
                      "config": {
                        "temperature": "bar",
                        "maxTokens": "bar",
                        "model": "bar"
                      }
                    }'
  /records/{recordId}/scores/{metricConfigId}:
    put:
      operationId: upsertScore
      summary: Upsert Score
      description: >-
        Create or update a Score for a given Record and MetricConfig. If a Score with the specified Record ID
        and MetricConfig ID already exists, it will be updated. Otherwise, a new Score will be created. The
        score provided should conform to the schema defined by the MetricConfig; otherwise, validation errors
        will be reported.
      parameters:
        - in: path
          name: recordId
          description: The ID of the Record to upsert the Score for.
          schema:
            type: string
            example: "777"
          required: true
        - in: path
          name: metricConfigId
          description: The ID of the MetricConfig.
          schema:
            type: string
            format: uuid
            example: a1b2c3d4-e5f6-7890-1234-567890abcdef
          required: true
      requestBody:
        content:
          application/json:
            schema:
              type: object
              properties:
                score:
                  type: object
                  additionalProperties: true
                  description: >-
                    The score of the Record, as arbitrary JSON. This data should ideally conform to the output
                    schema defined by the associated MetricConfig. If it doesn't, validation errors will be
                    captured in the `validationErrors` field.
              required:
                - score
            examples:
              Upsert simple score:
                value:
                  score:
                    value: true
                    reasoning: The response is correct
                summary: Upsert a simple score
                description: Creates or updates a Score with a basic score object.
              Upsert complex score:
                value:
                  score:
                    f1_score: 0.75
                    precision: 0.8
                    recall: 0.7
                    reasoning: Model performed well on typical cases.
                summary: Upsert a complex score
                description: >-
                  Creates or updates a Score with a more detailed, structured score, potentially including
                  multiple metrics and qualitative feedback.
              Upsert score needing validation:
                value:
                  score:
                    rating: excellent
                summary: Upsert score that may have validation issues
                description: >-
                  Example of a score that might not fully conform to the metric's output schema, which would
                  result in validationErrors in the response.
      responses:
        "200":
          description: Score upserted successfully.
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/Score"
              examples:
                Upserted simple score:
                  value:
                    recordId: "777"
                    metricConfigId: a1b2c3d4-e5f6-7890-1234-567890abcdef
                    score:
                      value: true
                      reasoning: The response is correct
                    validationErrors: []
                  summary: Successful simple score upsert
                  description: Response showing the full Score after a successful upsert with a simple score.
                Upserted score with validation errors:
                  value:
                    recordId: "778"
                    metricConfigId: b2c3d4e5-f6a7-8901-2345-67890abcdef0
                    score:
                      rating: excellent
                    validationErrors:
                      - path: /score/rating
                        message: Expected number, received string
                  summary: Upserted score with validation errors
                  description: >-
                    Response for a Score that was stored but whose `score` field had discrepancies with the
                    MetricConfig's output schema, detailed in `validationErrors`.
        "401":
          $ref: "#/components/responses/UnauthenticatedError"
        "500":
          $ref: "#/components/responses/ServiceError"
      x-codeSamples:
        - lang: JavaScript
          source: |-
            import Scorecard from 'scorecard-ai';

            const client = new Scorecard({
              apiKey: 'My API Key',
            });

            const score = await client.scores.upsert('a1b2c3d4-e5f6-7890-1234-567890abcdef', {
              recordId: '777',
              score: { value: true, reasoning: 'The response is correct' },
            });

            console.log(score.validationErrors);
        - lang: Python
          source: |-
            from scorecard_ai import Scorecard

            client = Scorecard(
                api_key="My API Key",
            )
            score = client.scores.upsert(
                metric_config_id="a1b2c3d4-e5f6-7890-1234-567890abcdef",
                record_id="777",
                score={
                    "value": True,
                    "reasoning": "The response is correct",
                },
            )
            print(score.validation_errors)
        - lang: cURL
          source: |-
            curl https://api2.scorecard.io/api/v2/records/$RECORD_ID/scores/$METRIC_CONFIG_ID \
                -X PUT \
                -H 'Content-Type: application/json' \
                -H "Authorization: Bearer $SCORECARD_API_KEY" \
                -d '{
                      "score": {
                        "value": "bar",
                        "reasoning": "bar"
                      }
                    }'
components:
  securitySchemes:
    ApiKeyAuth:
      type: http
      scheme: bearer
      bearerFormat: starts with ak_
  schemas:
    Project:
      type: object
      properties:
        id:
          type: string
          description: The ID of the Project.
        name:
          type:
            - string
            - "null"
          description: The name of the Project.
        description:
          type:
            - string
            - "null"
          description: The description of the Project.
      required:
        - id
        - name
        - description
      description: A Project in the Scorecard system.
    ApiError:
      type: object
      properties:
        code:
          type: string
        message:
          type: string
        details:
          type: object
          additionalProperties: true
          x-stainless-any: true
      required:
        - code
        - message
        - details
      description: An API error.
    Testset:
      type: object
      properties:
        id:
          type: string
          description: The ID of the Testset.
        name:
          type: string
          description: The name of the Testset.
        description:
          type: string
          description: The description of the Testset.
        jsonSchema:
          type: object
          description: The JSON schema for each Testcase in the Testset.
          additionalProperties: true
        fieldMapping:
          type: object
          properties:
            inputs:
              type: array
              items:
                type: string
              description: Fields that represent inputs to the AI system.
            expected:
              type: array
              items:
                type: string
              description: Fields that represent expected outputs.
            metadata:
              type: array
              items:
                type: string
              description: Fields that are not inputs or expected outputs.
          required:
            - inputs
            - expected
            - metadata
          description: >-
            Maps top-level keys of the Testcase schema to their roles (input/expected output). Unmapped fields
            are treated as metadata.
      required:
        - id
        - name
        - description
        - jsonSchema
        - fieldMapping
      description: >-
        A collection of Testcases that share the same schema.

        Each Testset defines the structure of its Testcases through a JSON schema.

        The `fieldMapping` object maps top-level keys of the Testcase schema to their roles (input/expected
        output).

        Fields not mentioned in the `fieldMapping` during creation or update are treated as metadata.


        ## JSON Schema validation constraints supported:


        - **Required fields** - Fields listed in the schema's `required` array must be present in Testcases.

        - **Type validation** - Values must match the specified type (string, number, boolean, null, integer,
        object, array).

        - **Enum validation** - Values must be one of the options specified in the `enum` array.

        - **Object property validation** - Properties of objects must conform to their defined schemas.

        - **Array item validation** - Items in arrays must conform to the `items` schema.

        - **Logical composition** - Values must conform to at least one schema in the `anyOf` array.


        Testcases that fail validation will still be stored, but will include `validationErrors` detailing the
        issues.

        Extra fields in the Testcase data that are not in the schema will be stored but are ignored during
        validation.
    Testcase:
      type: object
      properties:
        id:
          type: string
          description: The ID of the Testcase.
        testsetId:
          type: string
          description: The ID of the Testset this Testcase belongs to.
        jsonData:
          type: object
          additionalProperties: true
          description: The JSON data of the Testcase, which is validated against the Testset's schema.
        inputs:
          type: object
          additionalProperties: true
          description: >-
            Derived from data based on the Testset's fieldMapping. Contains all fields marked as inputs,
            including those with validation errors.
        expected:
          type: object
          additionalProperties: true
          description: >-
            Derived from data based on the Testset's fieldMapping. Contains all fields marked as expected
            outputs, including those with validation errors.
        validationErrors:
          type: array
          items:
            type: object
            properties:
              path:
                type: string
                description: JSON Pointer to the field with the validation error.
                example: /data/question
              message:
                type: string
                description: Human-readable error description.
                example: Required field missing
            required:
              - path
              - message
          description: >-
            Validation errors found in the Testcase data. If present, the Testcase doesn't fully conform to
            its Testset's schema.
      required:
        - id
        - testsetId
        - jsonData
        - inputs
        - expected
      description: >-
        A test case in the Scorecard system. Contains JSON data that is validated against the schema defined
        by its Testset.

        The `inputs` and `expected` fields are derived from the `data` field based on the Testset's
        `fieldMapping`, and include all mapped fields, including those with validation errors.

        Testcases are stored regardless of validation results, with any validation errors included in the
        `validationErrors` field.
    Metric:
      anyOf:
        - type: object
          properties:
            id:
              type: string
              description: The ID of the Metric.
            name:
              type: string
              description: The name of the Metric.
            description:
              type:
                - string
                - "null"
              default: null
              description: The description of the Metric.
            evalType:
              type: string
              const: ai
              description: AI-based evaluation type.
            guidelines:
              type: string
              description: Guidelines for AI evaluation on how to score the metric.
            promptTemplate:
              type: string
              description: >-
                The complete prompt template for AI evaluation. Should include placeholders for dynamic
                content.
            evalModelName:
              type: string
              default: gpt-4o
              description: The AI model to use for evaluation.
            temperature:
              type: number
              minimum: 0
              maximum: 2
              default: 0
              description: The temperature for AI evaluation (0-2).
            outputType:
              type: string
              const: int
              description: Integer output type.
            passingThreshold:
              type: integer
              minimum: 1
              maximum: 5
              default: 4
              description: The threshold for determining pass/fail from integer scores (1-5).
          required:
            - id
            - name
            - description
            - evalType
            - guidelines
            - promptTemplate
            - evalModelName
            - temperature
            - outputType
            - passingThreshold
          description: A Metric with AI evaluation and integer output.
          title: AI int metric
        - type: object
          properties:
            id:
              type: string
              description: The ID of the Metric.
            name:
              type: string
              description: The name of the Metric.
            description:
              type:
                - string
                - "null"
              default: null
              description: The description of the Metric.
            evalType:
              type: string
              const: human
              description: Human-based evaluation type.
            guidelines:
              type: string
              description: Guidelines for human evaluators.
            outputType:
              type: string
              const: int
              description: Integer output type.
            passingThreshold:
              type: integer
              minimum: 1
              maximum: 5
              default: 4
              description: The threshold for determining pass/fail from integer scores (1-5).
          required:
            - id
            - name
            - description
            - evalType
            - guidelines
            - outputType
            - passingThreshold
          description: A Metric with human evaluation and integer output.
          title: Human int metric
        - type: object
          properties:
            id:
              type: string
              description: The ID of the Metric.
            name:
              type: string
              description: The name of the Metric.
            description:
              type:
                - string
                - "null"
              default: null
              description: The description of the Metric.
            evalType:
              type: string
              const: heuristic
              description: Heuristic-based evaluation type.
            guidelines:
              type: string
              description: Guidelines for heuristic evaluation logic.
            outputType:
              type: string
              const: int
              description: Integer output type.
            passingThreshold:
              type: integer
              minimum: 1
              maximum: 5
              default: 4
              description: The threshold for determining pass/fail from integer scores (1-5).
          required:
            - id
            - name
            - description
            - evalType
            - guidelines
            - outputType
            - passingThreshold
          description: A Metric with heuristic evaluation and integer output.
          title: Heuristic int metric
        - type: object
          properties:
            id:
              type: string
              description: The ID of the Metric.
            name:
              type: string
              description: The name of the Metric.
            description:
              type:
                - string
                - "null"
              default: null
              description: The description of the Metric.
            evalType:
              type: string
              const: ai
              description: AI-based evaluation type.
            guidelines:
              type: string
              description: Guidelines for AI evaluation on how to score the metric.
            promptTemplate:
              type: string
              description: >-
                The complete prompt template for AI evaluation. Should include placeholders for dynamic
                content.
            evalModelName:
              type: string
              default: gpt-4o
              description: The AI model to use for evaluation.
            temperature:
              type: number
              minimum: 0
              maximum: 2
              default: 0
              description: The temperature for AI evaluation (0-2).
            outputType:
              type: string
              const: float
              description: Float output type (0-1).
            passingThreshold:
              type: number
              minimum: 0
              maximum: 1
              default: 0.9
              description: Threshold for determining pass/fail from float scores (0.0-1.0).
          required:
            - id
            - name
            - description
            - evalType
            - guidelines
            - promptTemplate
            - evalModelName
            - temperature
            - outputType
            - passingThreshold
          description: A Metric with AI evaluation and float output.
          title: AI float metric
        - type: object
          properties:
            id:
              type: string
              description: The ID of the Metric.
            name:
              type: string
              description: The name of the Metric.
            description:
              type:
                - string
                - "null"
              default: null
              description: The description of the Metric.
            evalType:
              type: string
              const: human
              description: Human-based evaluation type.
            guidelines:
              type: string
              description: Guidelines for human evaluators.
            outputType:
              type: string
              const: float
              description: Float output type (0-1).
            passingThreshold:
              type: number
              minimum: 0
              maximum: 1
              default: 0.9
              description: Threshold for determining pass/fail from float scores (0.0-1.0).
          required:
            - id
            - name
            - description
            - evalType
            - guidelines
            - outputType
            - passingThreshold
          description: A Metric with human evaluation and float output.
          title: Human float metric
        - type: object
          properties:
            id:
              type: string
              description: The ID of the Metric.
            name:
              type: string
              description: The name of the Metric.
            description:
              type:
                - string
                - "null"
              default: null
              description: The description of the Metric.
            evalType:
              type: string
              const: heuristic
              description: Heuristic-based evaluation type.
            guidelines:
              type: string
              description: Guidelines for heuristic evaluation logic.
            outputType:
              type: string
              const: float
              description: Float output type (0-1).
            passingThreshold:
              type: number
              minimum: 0
              maximum: 1
              default: 0.9
              description: Threshold for determining pass/fail from float scores (0.0-1.0).
          required:
            - id
            - name
            - description
            - evalType
            - guidelines
            - outputType
            - passingThreshold
          description: A Metric with heuristic evaluation and float output.
          title: Heuristic float metric
        - type: object
          properties:
            id:
              type: string
              description: The ID of the Metric.
            name:
              type: string
              description: The name of the Metric.
            description:
              type:
                - string
                - "null"
              default: null
              description: The description of the Metric.
            evalType:
              type: string
              const: ai
              description: AI-based evaluation type.
            guidelines:
              type: string
              description: Guidelines for AI evaluation on how to score the metric.
            promptTemplate:
              type: string
              description: >-
                The complete prompt template for AI evaluation. Should include placeholders for dynamic
                content.
            evalModelName:
              type: string
              default: gpt-4o
              description: The AI model to use for evaluation.
            temperature:
              type: number
              minimum: 0
              maximum: 2
              default: 0
              description: The temperature for AI evaluation (0-2).
            outputType:
              type: string
              const: boolean
              description: Boolean output type.
          required:
            - id
            - name
            - description
            - evalType
            - guidelines
            - promptTemplate
            - evalModelName
            - temperature
            - outputType
          description: A Metric with AI evaluation and boolean output.
          title: AI boolean metric
        - type: object
          properties:
            id:
              type: string
              description: The ID of the Metric.
            name:
              type: string
              description: The name of the Metric.
            description:
              type:
                - string
                - "null"
              default: null
              description: The description of the Metric.
            evalType:
              type: string
              const: human
              description: Human-based evaluation type.
            guidelines:
              type: string
              description: Guidelines for human evaluators.
            outputType:
              type: string
              const: boolean
              description: Boolean output type.
          required:
            - id
            - name
            - description
            - evalType
            - guidelines
            - outputType
          description: A Metric with human evaluation and boolean output.
          title: Human boolean metric
        - type: object
          properties:
            id:
              type: string
              description: The ID of the Metric.
            name:
              type: string
              description: The name of the Metric.
            description:
              type:
                - string
                - "null"
              default: null
              description: The description of the Metric.
            evalType:
              type: string
              const: heuristic
              description: Heuristic-based evaluation type.
            guidelines:
              type: string
              description: Guidelines for heuristic evaluation logic.
            outputType:
              type: string
              const: boolean
              description: Boolean output type.
          required:
            - id
            - name
            - description
            - evalType
            - guidelines
            - outputType
          description: A Metric with heuristic evaluation and boolean output.
          title: Heuristic boolean metric
      description: A Metric defines how to evaluate system outputs against expected results.
    Run:
      type: object
      properties:
        id:
          type: string
          description: The ID of the Run.
        testsetId:
          type:
            - string
            - "null"
          default: null
          description: The ID of the Testset this Run is testing.
        systemId:
          type:
            - string
            - "null"
          format: uuid
          description: The ID of the system this Run is using.
        systemVersionId:
          type:
            - string
            - "null"
          format: uuid
          description: The ID of the system version this Run is using.
        metricIds:
          type: array
          items:
            type: string
          description: The IDs of the metrics this Run is using.
        metricVersionIds:
          type: array
          items:
            type: string
            format: uuid
          description: The IDs of the metric versions this Run is using.
        numRecords:
          type: number
          description: The number of records in the Run.
        numExpectedRecords:
          type:
            - number
            - "null"
          description: >-
            The number of expected records in the Run. Determined by the number of testcases in the Run's
            Testset at the time of Run creation.
        numScores:
          type: number
          description: The number of completed scores in the Run so far.
        status:
          type: string
          enum:
            - pending
            - awaiting_execution
            - running_execution
            - awaiting_scoring
            - running_scoring
            - awaiting_human_scoring
            - completed
          description: The status of the Run.
      required:
        - id
        - testsetId
        - systemId
        - systemVersionId
        - metricIds
        - metricVersionIds
        - numRecords
        - numExpectedRecords
        - numScores
        - status
      description: A Run in the Scorecard system.
    Record:
      type: object
      properties:
        id:
          type: string
          description: The ID of the Record.
        runId:
          type: string
          description: The ID of the Run containing this Record.
        testcaseId:
          type: string
          description: The ID of the Testcase.
        inputs:
          type: object
          additionalProperties: true
          description: The actual inputs sent to the system, which should match the system's input schema.
        expected:
          type: object
          additionalProperties: true
          description: The expected outputs for the Testcase.
        outputs:
          type: object
          additionalProperties: true
          description: The actual outputs from the system.
      required:
        - id
        - runId
        - inputs
        - expected
        - outputs
      description: A record of a system execution in the Scorecard system.
    RecordWithScores:
      allOf:
        - $ref: "#/components/schemas/Record"
      properties:
        scores:
          type: array
          items:
            $ref: "#/components/schemas/Score"
          description: All scores associated with this record.
      required:
        - scores
      description: A record with all its associated scores.
    Annotation:
      type: object
      properties:
        id:
          type: string
          description: The ID of the Annotation.
        recordId:
          type: string
          description: The ID of the Record this Annotation belongs to.
        spanId:
          type:
            - string
            - "null"
          description: Optional span ID linking this annotation to a specific span.
        rating:
          type:
            - boolean
            - "null"
          description: "The rating of the annotation: true (positive), false (negative), or null (no rating)."
        comment:
          type:
            - string
            - "null"
          description: An optional text comment for the annotation.
        userId:
          type: string
          description: The ID of the user who created the annotation.
        createdAt:
          type: string
          description: The ISO 8601 timestamp when the annotation was created.
      required:
        - id
        - recordId
        - spanId
        - rating
        - comment
        - userId
        - createdAt
      description: An annotation on a Record, containing a rating (thumbs up/down) and/or a text comment.
    Score:
      type: object
      properties:
        recordId:
          type: string
          description: The ID of the Record this Score is for.
        metricConfigId:
          type: string
          format: uuid
          description: The ID of the MetricConfig this Score is for.
        score:
          type: object
          additionalProperties: true
          description: >-
            The score of the Record, as arbitrary JSON. This data should ideally conform to the output schema
            defined by the associated MetricConfig. If it doesn't, validation errors will be captured in the
            `validationErrors` field.
        validationErrors:
          type: array
          items:
            type: object
            properties:
              path:
                type: string
                description: JSON Pointer to the field with the validation error.
                example: /data/question
              message:
                type: string
                description: Human-readable error description.
                example: Required field missing
            required:
              - path
              - message
          description: >-
            Validation errors found in the Score data. If present, the Score doesn't fully conform to its
            MetricConfig's schema.
      required:
        - recordId
        - metricConfigId
        - score
      description: >-
        A Score represents the evaluation of a Record against a specific MetricConfig. The actual `score` is
        stored as flexible JSON. While any JSON is accepted, it is expected to conform to the output schema
        defined by the MetricConfig. Any discrepancies will be noted in the `validationErrors` field, but the
        Score will still be stored.
    System:
      type: object
      properties:
        id:
          type: string
          format: uuid
          description: The ID of the system.
        name:
          type: string
          description: The name of the system. Unique within the project.
        description:
          type: string
          default: ""
          description: The description of the system.
        productionVersion:
          $ref: "#/components/schemas/SystemVersion"
          description: The production version of the system.
        versions:
          type: array
          items:
            type: object
            properties:
              id:
                type: string
                format: uuid
                description: The ID of the system version.
              name:
                type: string
                description: The name of the system version.
            required:
              - id
              - name
            description: >-
              A SystemVersion defines the specific settings for a System Under Test.


              System versions contain parameter values that determine system behavior during evaluation.

              They are immutable snapshots - once created, they never change.


              When running evaluations, you reference a specific systemVersionId to establish which system
              version to test.
          description: The versions of the system.
      required:
        - id
        - name
        - description
        - productionVersion
        - versions
      description: |-
        A System Under Test (SUT).

        Systems are templates - to run evaluations, pair them with a SystemVersion that provides specific
        parameter values.
    SystemVersion:
      type: object
      properties:
        id:
          type: string
          format: uuid
          description: The ID of the system version.
        systemId:
          type: string
          format: uuid
          description: The ID of the system the system version belongs to.
        name:
          type: string
          description: The name of the system version.
        config:
          type: object
          additionalProperties: true
          description: The configuration of the system version.
      required:
        - id
        - systemId
        - name
        - config
      description: >-
        A SystemVersion defines the specific settings for a System Under Test.


        System versions contain parameter values that determine system behavior during evaluation.

        They are immutable snapshots - once created, they never change.


        When running evaluations, you reference a specific systemVersionId to establish which system version
        to test.
  responses:
    UnauthenticatedError:
      description: Error indicating that the request is not authenticated.
      content:
        application/json:
          schema:
            $ref: "#/components/schemas/ApiError"
          examples:
            Authentication failure:
              value:
                code: UNAUTHORIZED
                message: Invalid or missing authentication token
                details: {}
              summary: Authentication failure
              description: Error returned when authentication credentials are invalid or missing.
    ServiceError:
      description: An internal service error indicating an issue with the Scorecard service.
      content:
        application/json:
          schema:
            $ref: "#/components/schemas/ApiError"
          examples:
            Internal error:
              value:
                code: INTERNAL_ERROR
                message: An unexpected error occurred while processing your request.
                details: {}
              summary: Internal error
              description: Generic error when an unexpected internal issue occurs.