feat: add engine init endpoint (#888)

Author: louis-jan

cortex-js/src/domain/abstracts/engine.abstract.ts

@@ -10,6 +10,8 @@ export abstract class EngineExtension extends Extension {
 
   transformResponse?: Function;
 
+  initalized: boolean = false;
+
   abstract inference(
     dto: any,
     headers: Record<string, string>,

cortex-js/src/domain/abstracts/extension.abstract.ts

@@ -21,6 +21,9 @@ export abstract class Extension {
   /** @type {string} Extension's version. */
   version?: string;
 
+  /** @type {boolean} Whether the extension is initialized or not. */
+  initalized: boolean;
+
   /**
    * Called when the extension is loaded.
    * Any initialization logic for the extension should be put here.

cortex-js/src/domain/config/config.interface.ts

@@ -1,6 +1,5 @@
 export interface Config {
   dataFolderPath: string;
-  initialized: boolean;
   cortexCppHost: string;
   cortexCppPort: number;
 }

cortex-js/src/domain/models/download.interface.ts

@@ -69,4 +69,5 @@ export interface DownloadStateEvent {
 export enum DownloadType {
   Model = 'model',
   Miscelanous = 'miscelanous',
+  Engine = 'engine'
 }

cortex-js/src/extensions/anthropic.engine.ts

@@ -16,6 +16,7 @@ export default class AnthropicEngineExtension extends OAIEngineExtension {
   productName = 'Anthropic Inference Engine';
   description = 'This extension enables Anthropic chat completion API calls';
   version = '0.0.1';
+  initalized = true;
   apiKey?: string;
 
   constructor(
@@ -39,42 +40,45 @@ export default class AnthropicEngineExtension extends OAIEngineExtension {
     this.apiKey = configs?.apiKey;
   }
 
-  override async inference(dto: any, headers: Record<string, string>): Promise<stream.Readable | any> {
-    headers['x-api-key'] = this.apiKey as string
-    headers['Content-Type'] = 'application/json'
-    headers['anthropic-version'] = '2023-06-01'
-    return super.inference(dto, headers)
+  override async inference(
+    dto: any,
+    headers: Record<string, string>,
+  ): Promise<stream.Readable | any> {
+    headers['x-api-key'] = this.apiKey as string;
+    headers['Content-Type'] = 'application/json';
+    headers['anthropic-version'] = '2023-06-01';
+    return super.inference(dto, headers);
   }
 
   transformPayload = (data: any): any => {
     return _.pick(data, ['messages', 'model', 'stream', 'max_tokens']);
-  }
+  };
 
   transformResponse = (data: any): string => {
     // handling stream response
     if (typeof data === 'string' && data.trim().length === 0) {
-        return '';
+      return '';
     }
     if (typeof data === 'string' && data.startsWith('event: ')) {
-        return ''
+      return '';
     }
     if (typeof data === 'string' && data.startsWith('data: ')) {
       data = data.replace('data: ', '');
       const parsedData = JSON.parse(data);
       if (parsedData.type !== 'content_block_delta') {
-        return ''
+        return '';
       }
       const text = parsedData.delta?.text;
       //convert to have this format data.choices[0]?.delta?.content
       return JSON.stringify({
         choices: [
-            {
-                delta: {
-                content: text
-                }
-            }
-        ]
-      })
+          {
+            delta: {
+              content: text,
+            },
+          },
+        ],
+      });
     }
     // non-stream response
     if (data.content && data.content.length > 0 && data.content[0].text) {
@@ -88,8 +92,8 @@ export default class AnthropicEngineExtension extends OAIEngineExtension {
         ],
       });
     }
-  
+
     console.error('Invalid response format:', data);
     return '';
-  }
+  };
 }

cortex-js/src/extensions/groq.engine.ts

@@ -14,6 +14,7 @@ export default class GroqEngineExtension extends OAIEngineExtension {
   productName = 'Groq Inference Engine';
   description = 'This extension enables fast Groq chat completion API calls';
   version = '0.0.1';
+  initalized = true;
   apiKey?: string;
 
   constructor(

cortex-js/src/extensions/mistral.engine.ts

@@ -14,6 +14,7 @@ export default class MistralEngineExtension extends OAIEngineExtension {
   productName = 'Mistral Inference Engine';
   description = 'This extension enables Mistral chat completion API calls';
   version = '0.0.1';
+  initalized = true;
   apiKey?: string;
 
   constructor(

cortex-js/src/extensions/openai.engine.ts

@@ -14,6 +14,7 @@ export default class OpenAIEngineExtension extends OAIEngineExtension {
   productName = 'OpenAI Inference Engine';
   description = 'This extension enables OpenAI chat completion API calls';
   version = '0.0.1';
+  initalized = true;
   apiKey?: string;
 
   constructor(

cortex-js/src/infrastructure/commanders/engines/engines-init.command.ts

@@ -1,10 +1,10 @@
 import { CommandRunner, Option, SubCommand } from 'nest-commander';
 import { SetCommandContext } from '../decorators/CommandContext';
 import { ContextService } from '@/infrastructure/services/context/context.service';
-import { InitCliUsecases } from '../usecases/init.cli.usecases';
 import { Engines } from '../types/engine.interface';
 import { CortexUsecases } from '@/usecases/cortex/cortex.usecases';
 import { FileManagerService } from '@/infrastructure/services/file-manager/file-manager.service';
+import { EnginesUsecases } from '@/usecases/engines/engines.usecase';
 
 @SubCommand({
   name: '<name> init',
@@ -16,7 +16,7 @@ import { FileManagerService } from '@/infrastructure/services/file-manager/file-
 @SetCommandContext()
 export class EnginesInitCommand extends CommandRunner {
   constructor(
-    private readonly initUsecases: InitCliUsecases,
+    private readonly engineUsecases: EnginesUsecases,
     private readonly cortexUsecases: CortexUsecases,
     private readonly fileManagerService: FileManagerService,
     readonly contextService: ContextService,
@@ -31,7 +31,7 @@ export class EnginesInitCommand extends CommandRunner {
     const engine = passedParams[0];
     const params = passedParams.includes(Engines.llamaCPP)
       ? {
-          ...(await this.initUsecases.defaultInstallationOptions()),
+          ...(await this.engineUsecases.defaultInstallationOptions()),
           ...options,
         }
       : {};
@@ -43,7 +43,8 @@ export class EnginesInitCommand extends CommandRunner {
     if (await this.cortexUsecases.healthCheck(host, port)) {
       await this.cortexUsecases.stopCortex();
     }
-    return this.initUsecases
+    console.log(`Installing engine ${engine}...`);
+    return this.engineUsecases
       .installEngine(
         params,
         engine.includes('@') ? engine.split('@')[1] : 'latest',

cortex-js/src/infrastructure/commanders/models/model-pull.command.ts

@@ -12,9 +12,9 @@ import { ContextService } from '@/infrastructure/services/context/context.servic
 import { existsSync } from 'fs';
 import { join } from 'node:path';
 import { FileManagerService } from '@/infrastructure/services/file-manager/file-manager.service';
-import { InitCliUsecases } from '../usecases/init.cli.usecases';
 import { checkModelCompatibility } from '@/utils/model-check';
 import { Engines } from '../types/engine.interface';
+import { EnginesUsecases } from '@/usecases/engines/engines.usecase';
 
 @SubCommand({
   name: 'pull',
@@ -28,7 +28,7 @@ import { Engines } from '../types/engine.interface';
 export class ModelPullCommand extends CommandRunner {
   constructor(
     private readonly modelsCliUsecases: ModelsCliUsecases,
-    private readonly initUsecases: InitCliUsecases,
+    private readonly engineUsecases: EnginesUsecases,
     private readonly fileService: FileManagerService,
     readonly contextService: ContextService,
     private readonly telemetryUsecases: TelemetryUsecases,
@@ -60,7 +60,7 @@ export class ModelPullCommand extends CommandRunner {
       !existsSync(join(await this.fileService.getCortexCppEnginePath(), engine))
     ) {
       console.log('\n');
-      await this.initUsecases.installEngine(undefined, 'latest', engine);
+      await this.engineUsecases.installEngine(undefined, 'latest', engine);
     }
     this.telemetryUsecases.sendEvent(
       [