feat: Model Presets (#651)

Author: louis-jan

cortex-js/src/command.module.ts

@@ -26,6 +26,7 @@ import { MessagesModule } from './usecases/messages/messages.module';
 import { FileManagerModule } from './file-manager/file-manager.module';
 import { PSCommand } from './infrastructure/commanders/ps.command';
 import { KillCommand } from './infrastructure/commanders/kill.command';
+import { PresetCommand } from './infrastructure/commanders/presets.command';
 
 @Module({
   imports: [
@@ -52,6 +53,7 @@ import { KillCommand } from './infrastructure/commanders/kill.command';
     InitCommand,
     PSCommand,
     KillCommand,
+    PresetCommand,
 
     // Questions
     InitRunModeQuestions,

cortex-js/src/infrastructure/commanders/cortex-command.commander.ts

@@ -8,6 +8,7 @@ import { ModelPullCommand } from './models/model-pull.command';
 import { PSCommand } from './ps.command';
 import { KillCommand } from './kill.command';
 import pkg from '@/../package.json';
+import { PresetCommand } from './presets.command';
 
 interface CortexCommandOptions {
   version: boolean;
@@ -22,6 +23,7 @@ interface CortexCommandOptions {
     ModelPullCommand,
     PSCommand,
     KillCommand,
+    PresetCommand,
   ],
   description: 'Cortex CLI',
 })

cortex-js/src/infrastructure/commanders/models/model-list.command.ts

@@ -12,12 +12,21 @@ export class ModelListCommand extends CommandRunner {
 
   async run(_input: string[], option: ModelListOptions): Promise<void> {
     const models = await this.modelsCliUsecases.listAllModels();
-    option.format === 'table' ? console.table(models) : console.log(models);
+    option.format === 'table'
+      ? console.table(
+          models.map((e) => ({
+            id: e.id,
+            engine: e.engine,
+            format: e.format,
+            created: e.created,
+          })),
+        )
+      : console.log(models);
   }
 
   @Option({
     flags: '-f, --format <format>',
-    defaultValue: 'json',
+    defaultValue: 'table',
     description: 'Print models list in table or json format',
   })
   parseModelId(value: string) {

cortex-js/src/infrastructure/commanders/models/model-start.command.ts

@@ -10,6 +10,7 @@ import { CortexUsecases } from '@/usecases/cortex/cortex.usecases';
 
 type ModelStartOptions = {
   attach: boolean;
+  preset?: string;
 };
 @SubCommand({ name: 'start', description: 'Start a model by ID.' })
 export class ModelStartCommand extends CommandRunner {
@@ -34,7 +35,7 @@ export class ModelStartCommand extends CommandRunner {
 
     await this.cortexUsecases
       .startCortex(options.attach)
-      .then(() => this.modelsCliUsecases.startModel(modelId))
+      .then(() => this.modelsCliUsecases.startModel(modelId, options.preset))
       .then(console.log)
       .then(() => !options.attach && process.exit(0));
   }
@@ -63,4 +64,12 @@ export class ModelStartCommand extends CommandRunner {
   parseAttach() {
     return true;
   }
+
+  @Option({
+    flags: '-p, --preset <preset>',
+    description: 'Apply a chat preset to the chat session',
+  })
+  parseTemplate(value: string) {
+    return value;
+  }
 }

cortex-js/src/infrastructure/commanders/presets.command.ts

@@ -0,0 +1,23 @@
+import { FileManagerService } from '@/file-manager/file-manager.service';
+import { readdirSync } from 'fs';
+import { CommandRunner, SubCommand } from 'nest-commander';
+import { join } from 'path';
+
+@SubCommand({
+  name: 'presets',
+  description: 'Show all available presets',
+})
+export class PresetCommand extends CommandRunner {
+  constructor(private readonly fileService: FileManagerService) {
+    super();
+  }
+  async run(): Promise<void> {
+    return console.table(
+      readdirSync(
+        join(await this.fileService.getDataFolderPath(), `presets`),
+      ).map((e) => ({
+        preset: e.replace('.yaml', ''),
+      })),
+    );
+  }
+}

cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts

@@ -12,6 +12,7 @@ import { ModelsCliUsecases } from '../usecases/models.cli.usecases';
 
 type RunOptions = {
   threadId?: string;
+  preset?: string;
 };
 
 @SubCommand({
@@ -28,7 +29,7 @@ export class RunCommand extends CommandRunner {
     super();
   }
 
-  async run(input: string[], option?: RunOptions): Promise<void> {
+  async run(input: string[], options: RunOptions): Promise<void> {
     let modelId = input[0];
     if (!modelId) {
       try {
@@ -41,8 +42,8 @@ export class RunCommand extends CommandRunner {
 
     return this.cortexUsecases
       .startCortex(false, defaultCortexCppHost, defaultCortexCppPort)
-      .then(() => this.modelsCliUsecases.startModel(modelId))
-      .then(() => this.chatCliUsecases.chat(modelId, option?.threadId));
+      .then(() => this.modelsCliUsecases.startModel(modelId, options.preset))
+      .then(() => this.chatCliUsecases.chat(modelId, options.threadId));
   }
 
   @Option({
@@ -53,6 +54,14 @@ export class RunCommand extends CommandRunner {
     return value;
   }
 
+  @Option({
+    flags: '-p, --preset <preset>',
+    description: 'Apply a chat preset to the chat session',
+  })
+  parseTemplate(value: string) {
+    return value;
+  }
+
   modelInquiry = async () => {
     const models = await this.modelsCliUsecases.listAllModels();
     if (!models.length) throw 'No models found';

cortex-js/src/infrastructure/commanders/usecases/models.cli.usecases.ts

@@ -25,6 +25,10 @@ import { ModelTokenizer } from '../types/model-tokenizer.interface';
 import { HttpService } from '@nestjs/axios';
 import { firstValueFrom } from 'rxjs';
 import { StartModelSuccessDto } from '@/infrastructure/dtos/models/start-model-success.dto';
+import { FileManagerService } from '@/file-manager/file-manager.service';
+import { join } from 'path';
+import { load } from 'js-yaml';
+import { existsSync, readFileSync } from 'node:fs';
 
 const AllQuantizations = [
   'Q3_K_S',
@@ -56,15 +60,24 @@ export class ModelsCliUsecases {
     @Inject(InquirerService)
     private readonly inquirerService: InquirerService,
     private readonly httpService: HttpService,
+    private readonly fileManagerService: FileManagerService,
   ) {}
 
   /**
    * Start a model by ID
    * @param modelId
    */
-  async startModel(modelId: string): Promise<StartModelSuccessDto> {
+  async startModel(
+    modelId: string,
+    preset?: string,
+  ): Promise<StartModelSuccessDto> {
+    const parsedPreset = await this.parsePreset(preset);
     return this.getModelOrStop(modelId)
-      .then(() => this.modelsUsecases.startModel(modelId))
+      .then((model) => ({
+        ...model.settings,
+        ...parsedPreset,
+      }))
+      .then((settings) => this.modelsUsecases.startModel(modelId, settings))
       .catch(() => {
         return {
           modelId: modelId,
@@ -164,7 +177,7 @@ export class ModelsCliUsecases {
       ],
       id: modelId,
       name: modelId,
-      version: '',
+      version: '1.0.0',
       format: ModelFormat.GGUF,
       description: '',
       settings: {
@@ -440,4 +453,16 @@ export class ModelsCliUsecases {
   private getRepoModelsUrl(repoId: string, tree?: string): string {
     return `https://huggingface.co/api/models/${repoId}${tree ? `/tree/${tree}` : ''}`;
   }
+
+  private async parsePreset(preset?: string): Promise<object> {
+    const presetPath = join(
+      await this.fileManagerService.getDataFolderPath(),
+      'presets',
+      `${preset}.yaml`,
+    );
+    if (!preset || !existsSync(presetPath)) return {};
+    return preset
+      ? (load(readFileSync(join(presetPath), 'utf-8')) as object)
+      : {};
+  }
 }

cortex-js/src/infrastructure/controllers/chat.controller.ts

@@ -2,14 +2,7 @@ import { Body, Controller, Post, Headers, Res, HttpCode } from '@nestjs/common';
 import { CreateChatCompletionDto } from '@/infrastructure/dtos/chat/create-chat-completion.dto';
 import { ChatUsecases } from '@/usecases/chat/chat.usecases';
 import { Response } from 'express';
-import {
-  ApiCreatedResponse,
-  ApiExtraModels,
-  ApiOperation,
-  ApiTags,
-  getSchemaPath,
-  ApiResponse
-} from '@nestjs/swagger';
+import { ApiOperation, ApiTags, ApiResponse } from '@nestjs/swagger';
 import { ChatCompletionResponseDto } from '../dtos/chat/chat-completion-response.dto';
 
 @ApiTags('Inference')
@@ -19,7 +12,7 @@ export class ChatController {
 
   @ApiOperation({
     summary: 'Create chat completion',
-    description: "Creates a model response for the given conversation.",
+    description: 'Creates a model response for the given conversation.',
   })
   @HttpCode(200)
   @ApiResponse({

cortex-js/src/infrastructure/dtos/assistants/create-assistant.dto.ts

@@ -5,7 +5,7 @@ import {
   AssistantMetadata,
 } from '@/domain/models/assistant.interface';
 import { AssistantToolDto } from './assistant-tool.dto';
-import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
+import { ApiProperty } from '@nestjs/swagger';
 
 export class CreateAssistantDto implements Partial<Assistant> {
   @ApiProperty({

cortex-js/src/infrastructure/dtos/assistants/delete-assistant.dto.ts

@@ -1,22 +1,22 @@
 import { ApiProperty } from '@nestjs/swagger';
 
 export class DeleteAssistantResponseDto {
-    @ApiProperty({
-        example: 'assistant_123',
-        description: 'The identifier of the assistant that was deleted.'
-    })
-    id: string;
+  @ApiProperty({
+    example: 'assistant_123',
+    description: 'The identifier of the assistant that was deleted.',
+  })
+  id: string;
 
-    @ApiProperty({
-        example: 'assistant',
-        description: 'Type of the object, indicating it\'s a assistant.',
-        default: 'assistant'
-    })
-    object: string;
+  @ApiProperty({
+    example: 'assistant',
+    description: "Type of the object, indicating it's a assistant.",
+    default: 'assistant',
+  })
+  object: string;
 
-    @ApiProperty({
-        example: true,
-        description: 'Indicates whether the assistant was successfully deleted.'
-    })
-    deleted: boolean;
+  @ApiProperty({
+    example: true,
+    description: 'Indicates whether the assistant was successfully deleted.',
+  })
+  deleted: boolean;
 }