feat: cortex supports remote engines

Author: louis-jan

cortex-js/src/domain/abstracts/oai.abstract.ts

@@ -5,6 +5,7 @@ import { firstValueFrom } from 'rxjs';
 
 export abstract class OAIEngineExtension extends EngineExtension {
   abstract apiUrl: string;
+  abstract apiKey?: string;
 
   constructor(protected readonly httpService: HttpService) {
     super();
@@ -21,7 +22,9 @@ export abstract class OAIEngineExtension extends EngineExtension {
       this.httpService.post(this.apiUrl, createChatDto, {
         headers: {
           'Content-Type': headers['content-type'] ?? 'application/json',
-          Authorization: headers['authorization'],
+          Authorization: this.apiKey
+            ? `Bearer ${this.apiKey}`
+            : headers['authorization'],
         },
         responseType: stream ? 'stream' : 'json',
       }),

cortex-js/src/extensions/groq.engine.ts

@@ -13,6 +13,7 @@ export default class GroqEngineExtension extends OAIEngineExtension {
   productName = 'Groq Inference Engine';
   description = 'This extension enables fast Groq chat completion API calls';
   version = '0.0.1';
+  apiKey?: string;
 
   constructor(
     protected readonly httpService: HttpService,
@@ -25,6 +26,8 @@ export default class GroqEngineExtension extends OAIEngineExtension {
     const configs = (await this.configsUsecases.getGroupConfigs(
       this.name,
     )) as unknown as { apiKey: string };
+
+    this.apiKey = configs?.apiKey
     if (!configs?.apiKey)
       await this.configsUsecases.saveConfig('apiKey', '', this.name);
   }

cortex-js/src/extensions/mistral.engine.ts

@@ -13,6 +13,7 @@ export default class MistralEngineExtension extends OAIEngineExtension {
   productName = 'Mistral Inference Engine';
   description = 'This extension enables Mistral chat completion API calls';
   version = '0.0.1';
+  apiKey?: string;
 
   constructor(
     protected readonly httpService: HttpService,
@@ -25,6 +26,7 @@ export default class MistralEngineExtension extends OAIEngineExtension {
     const configs = (await this.configsUsecases.getGroupConfigs(
       this.name,
     )) as unknown as { apiKey: string };
+    this.apiKey = configs?.apiKey;
     if (!configs?.apiKey)
       await this.configsUsecases.saveConfig('apiKey', '', this.name);
   }

cortex-js/src/extensions/openai.engine.ts

@@ -13,6 +13,7 @@ export default class OpenAIEngineExtension extends OAIEngineExtension {
   productName = 'OpenAI Inference Engine';
   description = 'This extension enables OpenAI chat completion API calls';
   version = '0.0.1';
+  apiKey?: string;
 
   constructor(
     protected readonly httpService: HttpService,
@@ -25,6 +26,7 @@ export default class OpenAIEngineExtension extends OAIEngineExtension {
     const configs = (await this.configsUsecases.getGroupConfigs(
       this.name,
     )) as unknown as { apiKey: string };
+    this.apiKey = configs?.apiKey;
     if (!configs?.apiKey)
       await this.configsUsecases.saveConfig('apiKey', '', this.name);
   }

cortex-js/src/infrastructure/commanders/usecases/chat.cli.usecases.ts

@@ -192,7 +192,7 @@ export class ChatCliUsecases {
                 const toParse = cachedLines + line;
                 if (!line.includes('data: [DONE]')) {
                   const data = JSON.parse(toParse.replace('data: ', ''));
-                  content += data.choices[0]?.delta?.content ?? '';
+                  content = data.choices[0]?.delta?.content ?? '';
 
                   if (content.startsWith('assistant: ')) {
                     content = content.replace('assistant: ', '');
@@ -209,8 +209,10 @@ export class ChatCliUsecases {
             }
           });
         })
-        .catch(() => {
-          stdout.write('Something went wrong! Please check model status.\n');
+        .catch((e: any) => {
+          stdout.write(
+            `Something went wrong! Please check model status.\n${e.message}\n`,
+          );
           if (attach) rl.prompt();
           else rl.close();
         });

cortex-js/src/infrastructure/providers/cortex/cortex.provider.ts

@@ -21,6 +21,7 @@ export default class CortexProvider extends OAIEngineExtension {
   description =
     'This extension enables chat completion API calls using the Cortex engine';
   version = '0.0.1';
+  apiKey?: string | undefined;
 
   private loadModelUrl = `http://${defaultCortexCppHost}:${defaultCortexCppPort}/inferences/server/loadmodel`;
   private unloadModelUrl = `http://${defaultCortexCppHost}:${defaultCortexCppPort}/inferences/server/unloadmodel`;