feat(api): add test evaluation method

Author: stainless-app[bot]

.stats.yml

@@ -1,3 +1,3 @@
-configured_endpoints: 18
+configured_endpoints: 19
 openapi_spec_hash: 6a7cafc3d32e6701301e925175ab6614
-config_hash: 6dcf08c4324405f152d1da9fc11ab04a
+config_hash: 00442fdab71911b02ab1e10f9ec05b79

api.md

@@ -122,3 +122,13 @@ Types:
 Methods:
 
 - <code title="post /storage/presigned-url">client.storage.presignedURL.<a href="./src/resources/storage/presigned-url.ts">create</a>({ ...params }) -> PresignedURLCreateResponse</code>
+
+# Tests
+
+Types:
+
+- <code><a href="./src/resources/tests.ts">TestEvaluateResponse</a></code>
+
+Methods:
+
+- <code title="post /tests/{testId}/evaluate">client.tests.<a href="./src/resources/tests.ts">evaluate</a>(testID, { ...params }) -> TestEvaluateResponse</code>

src/client.ts

@@ -17,6 +17,7 @@ import * as Errors from './core/error';
 import * as Uploads from './core/uploads';
 import * as API from './resources/index';
 import { APIPromise } from './core/api-promise';
+import { TestEvaluateParams, TestEvaluateResponse, Tests } from './resources/tests';
 import { CommitRetrieveResponse, Commits } from './resources/commits/commits';
 import {
   InferencePipelineRetrieveParams,
@@ -724,12 +725,14 @@ export class Openlayer {
   commits: API.Commits = new API.Commits(this);
   inferencePipelines: API.InferencePipelines = new API.InferencePipelines(this);
   storage: API.Storage = new API.Storage(this);
+  tests: API.Tests = new API.Tests(this);
 }
 
 Openlayer.Projects = Projects;
 Openlayer.Commits = Commits;
 Openlayer.InferencePipelines = InferencePipelines;
 Openlayer.Storage = Storage;
+Openlayer.Tests = Tests;
 
 export declare namespace Openlayer {
   export type RequestOptions = Opts.RequestOptions;
@@ -753,4 +756,10 @@ export declare namespace Openlayer {
   };
 
   export { Storage as Storage };
+
+  export {
+    Tests as Tests,
+    type TestEvaluateResponse as TestEvaluateResponse,
+    type TestEvaluateParams as TestEvaluateParams,
+  };
 }

src/resources/index.ts

@@ -16,3 +16,4 @@ export {
   type ProjectListParams,
 } from './projects/projects';
 export { Storage } from './storage/storage';
+export { Tests, type TestEvaluateResponse, type TestEvaluateParams } from './tests';

src/resources/tests.ts

@@ -0,0 +1,100 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../core/resource';
+import { APIPromise } from '../core/api-promise';
+import { RequestOptions } from '../internal/request-options';
+import { path } from '../internal/utils/path';
+
+export class Tests extends APIResource {
+  /**
+   * Triggers one-off evaluation of a specific monitoring test for a custom timestamp
+   * range. This allows evaluating tests for historical data or custom time periods
+   * outside the regular evaluation window schedule. It also allows overwriting the
+   * existing test results.
+   *
+   * @example
+   * ```ts
+   * const response = await client.tests.evaluate(
+   *   '182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e',
+   *   { endTimestamp: 1700006400, startTimestamp: 1699920000 },
+   * );
+   * ```
+   */
+  evaluate(
+    testID: string,
+    body: TestEvaluateParams,
+    options?: RequestOptions,
+  ): APIPromise<TestEvaluateResponse> {
+    return this._client.post(path`/tests/${testID}/evaluate`, { body, ...options });
+  }
+}
+
+export interface TestEvaluateResponse {
+  message: string;
+
+  /**
+   * Number of inference pipelines the test was queued for evaluation on
+   */
+  pipelineCount: number;
+
+  /**
+   * The end timestamp you requested (in seconds)
+   */
+  requestedEndTimestamp: number;
+
+  /**
+   * The start timestamp you requested (in seconds)
+   */
+  requestedStartTimestamp: number;
+
+  /**
+   * Array of background task information for each pipeline evaluation
+   */
+  tasks: Array<TestEvaluateResponse.Task>;
+}
+
+export namespace TestEvaluateResponse {
+  export interface Task {
+    /**
+     * ID of the inference pipeline this task is for
+     */
+    pipelineId: string;
+
+    /**
+     * ID of the background task
+     */
+    taskResultId: string;
+
+    /**
+     * URL to check the status of this background task
+     */
+    taskResultUrl: string;
+  }
+}
+
+export interface TestEvaluateParams {
+  /**
+   * End timestamp in seconds (Unix epoch)
+   */
+  endTimestamp: number;
+
+  /**
+   * Start timestamp in seconds (Unix epoch)
+   */
+  startTimestamp: number;
+
+  /**
+   * ID of the inference pipeline to evaluate. If not provided, all inference
+   * pipelines the test applies to will be evaluated.
+   */
+  inferencePipelineId?: string;
+
+  /**
+   * Whether to overwrite existing test results
+   */
+  overwriteResults?: boolean;
+}
+
+export declare namespace Tests {
+  export { type TestEvaluateResponse as TestEvaluateResponse, type TestEvaluateParams as TestEvaluateParams };
+}

tests/api-resources/tests.test.ts

@@ -0,0 +1,33 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import Openlayer from 'openlayer';
+
+const client = new Openlayer({
+  apiKey: 'My API Key',
+  baseURL: process.env['TEST_API_BASE_URL'] ?? 'http://127.0.0.1:4010',
+});
+
+describe('resource tests', () => {
+  test('evaluate: only required params', async () => {
+    const responsePromise = client.tests.evaluate('182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e', {
+      endTimestamp: 1700006400,
+      startTimestamp: 1699920000,
+    });
+    const rawResponse = await responsePromise.asResponse();
+    expect(rawResponse).toBeInstanceOf(Response);
+    const response = await responsePromise;
+    expect(response).not.toBeInstanceOf(Response);
+    const dataAndResponse = await responsePromise.withResponse();
+    expect(dataAndResponse.data).toBe(response);
+    expect(dataAndResponse.response).toBe(rawResponse);
+  });
+
+  test('evaluate: required and optional params', async () => {
+    const response = await client.tests.evaluate('182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e', {
+      endTimestamp: 1700006400,
+      startTimestamp: 1699920000,
+      inferencePipelineId: '123e4567-e89b-12d3-a456-426614174000',
+      overwriteResults: false,
+    });
+  });
+});