feat: add logic to create scorer-specs W-22799005

src/agentScorer.ts

@@ -0,0 +1,113 @@
+/*
+ * Copyright 2026, Salesforce, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { mkdir, writeFile } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { stringify } from 'yaml';
+import { ScorerSpec } from './types.js';
+
+/** Default template for a Number (measurement) scorer. */
+const NUMBER_SCORER_TEMPLATE: ScorerSpec = {
+  name: 'My_Custom_Scorer',
+  description: 'A custom numeric scorer',
+  inputScope: 'Session',
+  dataType: 'Number',
+  version: {
+    versionNumber: 1,
+    status: 'Draft',
+    label: 'My Custom Scorer',
+    description: 'Evaluates session quality on a 0-5 scale',
+    agentApiName: 'My_Agent',
+    isActive: false,
+    engine: {
+      engineType: 'PromptTemplate',
+      engineRef: 'My_Scorer_Prompt_Template',
+    },
+    outputEnumValues: [
+      { value: '0', outcomeType: 'Fail', isFallback: false },
+      { value: '1', outcomeType: 'Fail', isFallback: false },
+      { value: '2', outcomeType: 'Fail', isFallback: false },
+      { value: '3', outcomeType: 'Pass', isFallback: true },
+      { value: '4', outcomeType: 'Pass', isFallback: false },
+      { value: '5', outcomeType: 'Pass', isFallback: false },
+    ],
+    valueSpecification: {
+      min: 0,
+      max: 5,
+      step: 1,
+      threshold: 3,
+    },
+  },
+};
+
+/** Default template for a Text (multilabel) scorer. */
+const TEXT_SCORER_TEMPLATE: ScorerSpec = {
+  name: 'My_Custom_Scorer',
+  description: 'A custom text classifier scorer',
+  inputScope: 'Session',
+  dataType: 'Text',
+  version: {
+    versionNumber: 1,
+    status: 'Draft',
+    label: 'My Custom Scorer',
+    description: 'Classifies sessions by category',
+    agentApiName: 'My_Agent',
+    isActive: false,
+    engine: {
+      engineType: 'PromptTemplate',
+      engineRef: 'My_Scorer_Prompt_Template',
+    },
+    outputEnumValues: [
+      { value: 'category_a', outcomeType: 'NotApplicable', isFallback: false },
+      { value: 'category_b', outcomeType: 'NotApplicable', isFallback: false },
+      { value: 'NOT_FOUND', outcomeType: 'NotApplicable', isFallback: true },
+    ],
+  },
+};
+
+export class AgentScorer {
+  /**
+   * Write a scorer spec YAML template to the given output file.
+   *
+   * @param outputFile - Destination file path.
+   * @param dataType - Whether to emit a Number or Text starter template.
+   * @param overrides - Optional values to override in the template before writing.
+   */
+  public static async writeScorerSpecTemplate(
+    outputFile: string,
+    dataType: 'Number' | 'Text' = 'Number',
+    overrides: { name?: string; agentApiName?: string } = {}
+  ): Promise<void> {
+    const base = dataType === 'Text' ? TEXT_SCORER_TEMPLATE : NUMBER_SCORER_TEMPLATE;
+    const template: ScorerSpec = {
+      ...base,
+      ...(overrides.name && { name: overrides.name }),
+      version: {
+        ...base.version,
+        ...(overrides.name && { label: overrides.name }),
+        ...(overrides.agentApiName && { agentApiName: overrides.agentApiName }),
+      },
+    };
+    const yml = stringify(template, undefined, { minContentWidth: 0, lineWidth: 0 });
+    await mkdir(dirname(outputFile), { recursive: true });
+    await writeFile(outputFile, yml);
+  }
+
+  /** Default output path for a scorer spec YAML, mirroring testspec naming. */
+  public static defaultSpecPath(scorerName: string): string {
+    return join('specs', `${scorerName}-scorerSpec.yaml`);
+  }
+}

src/index.ts

@@ -78,6 +78,15 @@ export {
   type AiTestCaseScorer,
   type AiConversationTurnXml,
 
+  // Custom Scorer Spec Types
+  type ScorerSpec,
+  type ScorerEngineType,
+  type ScorerInputScope,
+  type ScorerDataType,
+  type ScorerOutcomeType,
+  type ScorerOutputEnumValue,
+  type ScorerValueSpecification,
+
   // Agentforce Studio Testing Types
   type AgentforceStudioTestStartResponse,
   type AgentforceStudioTestStatusResponse,
@@ -139,6 +148,7 @@ export {
   isNgtScorerName,
 } from './ngtScorerCatalog';
 export { AgentTest, AgentTestCreateLifecycleStages } from './agentTest';
+export { AgentScorer } from './agentScorer';
 export { ProductionAgent } from './agents/productionAgent';
 export { ScriptAgent } from './agents/scriptAgent';
 export { AgentBase } from './agents/agentBase';

src/types.ts

@@ -615,6 +615,47 @@ export type AiTestingDefinition = {
   testCase: AiTestCase[];
 };
 
+// ====================================================
+//          Custom Scorer Spec (YAML) Types
+// ====================================================
+export type ScorerEngineType = 'PromptTemplate' | 'Expression';
+export type ScorerInputScope = 'Session' | 'Interaction' | 'Moment';
+export type ScorerDataType = 'Number' | 'Text';
+export type ScorerOutcomeType = 'Pass' | 'Fail' | 'NotApplicable';
+export type ScorerOutputEnumValue = {
+  value: string;
+  outcomeType: ScorerOutcomeType;
+  isFallback: boolean;
+};
+export type ScorerValueSpecification = {
+  min: number;
+  max: number;
+  step?: number;
+  threshold?: number;
+};
+
+export type ScorerSpec = {
+  name: string;
+  description?: string;
+  inputScope: ScorerInputScope;
+  dataType: ScorerDataType;
+  version: {
+    versionNumber: number;
+    status: 'Draft' | 'Available' | 'Archived';
+    label: string;
+    description?: string;
+    agentApiName: string;
+    isActive: boolean;
+    engine: {
+      engineType: ScorerEngineType;
+      engineRef?: string;
+      engineValue?: string;
+    };
+    outputEnumValues: ScorerOutputEnumValue[];
+    valueSpecification?: ScorerValueSpecification;
+  };
+};
+
 // ====================================================
 //               Agent Preview Types
 // ====================================================

test/agentScorer.test.ts

@@ -0,0 +1,111 @@
+/*
+ * Copyright 2026, Salesforce, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { readFile } from 'node:fs/promises';
+import { expect } from 'chai';
+import { parse } from 'yaml';
+import { AgentScorer } from '../src/agentScorer';
+import type { ScorerSpec } from '../src/types';
+
+describe('AgentScorer', () => {
+  describe('defaultSpecPath', () => {
+    it('returns specs/<name>-scorerSpec.yaml', () => {
+      expect(AgentScorer.defaultSpecPath('My_Scorer')).to.equal(join('specs', 'My_Scorer-scorerSpec.yaml'));
+    });
+  });
+
+  describe('writeScorerSpecTemplate', () => {
+    const outputFile = join(tmpdir(), `agentScorer-test-${Date.now()}.yaml`);
+
+    it('writes a valid Number template by default', async () => {
+      await AgentScorer.writeScorerSpecTemplate(outputFile);
+      const raw = await readFile(outputFile, 'utf-8');
+      const spec = parse(raw) as ScorerSpec;
+
+      expect(spec.dataType).to.equal('Number');
+      expect(spec.inputScope).to.equal('Session');
+      expect(spec.version.status).to.equal('Draft');
+      expect(spec.version.engine.engineType).to.equal('PromptTemplate');
+      expect(spec.version.outputEnumValues).to.have.length.greaterThan(0);
+      expect(spec.version.valueSpecification).to.exist;
+    });
+
+    it('writes a valid Text template when dataType is Text', async () => {
+      const textFile = join(tmpdir(), `agentScorer-text-test-${Date.now()}.yaml`);
+      await AgentScorer.writeScorerSpecTemplate(textFile, 'Text');
+      const raw = await readFile(textFile, 'utf-8');
+      const spec = parse(raw) as ScorerSpec;
+
+      expect(spec.dataType).to.equal('Text');
+      expect(spec.version.valueSpecification).to.be.undefined;
+      expect(spec.version.outputEnumValues.every((v) => v.outcomeType === 'NotApplicable')).to.be.true;
+    });
+
+    it('applies --name override to name and label', async () => {
+      const namedFile = join(tmpdir(), `agentScorer-name-test-${Date.now()}.yaml`);
+      await AgentScorer.writeScorerSpecTemplate(namedFile, 'Number', { name: 'Sentiment_Scorer' });
+      const raw = await readFile(namedFile, 'utf-8');
+      const spec = parse(raw) as ScorerSpec;
+
+      expect(spec.name).to.equal('Sentiment_Scorer');
+      expect(spec.version.label).to.equal('Sentiment_Scorer');
+    });
+
+    it('applies --agent-api-name override', async () => {
+      const agentFile = join(tmpdir(), `agentScorer-agent-test-${Date.now()}.yaml`);
+      await AgentScorer.writeScorerSpecTemplate(agentFile, 'Number', { agentApiName: 'Resort_Agent' });
+      const raw = await readFile(agentFile, 'utf-8');
+      const spec = parse(raw) as ScorerSpec;
+
+      expect(spec.version.agentApiName).to.equal('Resort_Agent');
+    });
+
+    it('applies both overrides at once', async () => {
+      const bothFile = join(tmpdir(), `agentScorer-both-test-${Date.now()}.yaml`);
+      await AgentScorer.writeScorerSpecTemplate(bothFile, 'Text', {
+        name: 'Language_Classifier',
+        agentApiName: 'My_Agent',
+      });
+      const raw = await readFile(bothFile, 'utf-8');
+      const spec = parse(raw) as ScorerSpec;
+
+      expect(spec.name).to.equal('Language_Classifier');
+      expect(spec.version.label).to.equal('Language_Classifier');
+      expect(spec.version.agentApiName).to.equal('My_Agent');
+      expect(spec.dataType).to.equal('Text');
+    });
+
+    it('Text template has exactly one fallback value', async () => {
+      const textFile = join(tmpdir(), `agentScorer-fallback-test-${Date.now()}.yaml`);
+      await AgentScorer.writeScorerSpecTemplate(textFile, 'Text');
+      const raw = await readFile(textFile, 'utf-8');
+      const spec = parse(raw) as ScorerSpec;
+
+      const fallbacks = spec.version.outputEnumValues.filter((v) => v.isFallback);
+      expect(fallbacks).to.have.length(1);
+    });
+
+    it('Number template has exactly one fallback value', async () => {
+      await AgentScorer.writeScorerSpecTemplate(outputFile, 'Number');
+      const raw = await readFile(outputFile, 'utf-8');
+      const spec = parse(raw) as ScorerSpec;
+
+      const fallbacks = spec.version.outputEnumValues.filter((v) => v.isFallback);
+      expect(fallbacks).to.have.length(1);
+    });
+  });
+});