Merge pull request #557 from janhq/chore/clean-up-fetch

chore: clean up node-fetch

Author: namchuai

cortex-js/package.json

@@ -38,7 +38,6 @@
     "class-validator": "^0.14.1",
     "cli-progress": "^3.12.0",
     "nest-commander": "^3.13.0",
-    "node-fetch": "2",
     "readline": "^1.3.0",
     "reflect-metadata": "^0.2.0",
     "rxjs": "^7.8.1",

cortex-js/src/domain/abstracts/oai.abstract.ts

@@ -1,38 +1,52 @@
 /* eslint-disable @typescript-eslint/no-unused-vars */
+import { HttpService } from '@nestjs/axios';
 import { EngineExtension } from './engine.abstract';
 
 export abstract class OAIEngineExtension extends EngineExtension {
   abstract apiUrl: string;
 
+  constructor(protected readonly httpService: HttpService) {
+    super();
+  }
+
   async inference(
     createChatDto: any,
     headers: Record<string, string>,
     res: any,
   ) {
-    // eslint-disable-next-line @typescript-eslint/no-var-requires
-    const fetch = require('node-fetch');
-    const response = await fetch(this.apiUrl, {
-      method: 'POST',
-      headers: {
-        'Content-Type': headers['content-type'] ?? 'application/json',
-        Authorization: headers['authorization'],
-      },
-      body: JSON.stringify(createChatDto),
-    });
+    if (createChatDto.stream === true) {
+      const response = await this.httpService
+        .post(this.apiUrl, createChatDto, {
+          headers: {
+            'Content-Type': headers['content-type'] ?? 'application/json',
+            Authorization: headers['authorization'],
+          },
+          responseType: 'stream',
+        })
+        .toPromise();
+
+      res.writeHead(200, {
+        'Content-Type': 'text/event-stream',
+        'Cache-Control': 'no-cache',
+        Connection: 'keep-alive',
+        'Access-Control-Allow-Origin': '*',
+      });
 
-    res.writeHead(200, {
-      'Content-Type':
-        createChatDto.stream === true
-          ? 'text/event-stream'
-          : 'application/json',
-      'Cache-Control': 'no-cache',
-      Connection: 'keep-alive',
-      'Access-Control-Allow-Origin': '*',
-    });
+      response?.data.pipe(res);
+    } else {
+      const response = await this.httpService
+        .post(this.apiUrl, createChatDto, {
+          headers: {
+            'Content-Type': headers['content-type'] ?? 'application/json',
+            Authorization: headers['authorization'],
+          },
+        })
+        .toPromise();
 
-    response.body.pipe(res);
+      res.json(response?.data);
+    }
   }
 
-  async loadModel(loadModel: any): Promise<void> {}
-  async unloadModel(modelId: string): Promise<void> {}
+  async loadModel(_loadModel: any): Promise<void> {}
+  async unloadModel(_modelId: string): Promise<void> {}
 }

cortex-js/src/infrastructure/providers/cortex/cortex.module.ts

@@ -1,7 +1,9 @@
 import { Module } from '@nestjs/common';
 import CortexProvider from './cortex.provider';
+import { HttpModule } from '@nestjs/axios';
 
 @Module({
+  imports: [HttpModule],
   providers: [
     {
       provide: 'CORTEX_PROVIDER',

cortex-js/src/infrastructure/providers/cortex/cortex.provider.ts

@@ -4,6 +4,8 @@ import { PromptTemplate } from '@/domain/models/prompt-template.interface';
 import { basename, join, resolve } from 'path';
 import { Model } from '@/domain/models/model.interface';
 import { ConfigService } from '@nestjs/config';
+import { HttpService } from '@nestjs/axios';
+
 /**
  * A class that implements the InferenceExtension interface from the @janhq/core package.
  * The class provides methods for initializing and stopping a model, and for making inference requests.
@@ -14,13 +16,17 @@ const NITRO_DEFAULT_PORT = 3928;
 const NITRO_HTTP_SERVER_URL = `http://${LOCAL_HOST}:${NITRO_DEFAULT_PORT}`;
 const LOAD_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/server/loadmodel`;
 const UNLOAD_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/server/unloadmodel`;
+
 @Injectable()
 export default class CortexProvider extends OAIEngineExtension {
   provider: string = 'cortex';
   apiUrl = 'http://127.0.0.1:3928/inferences/server/chat_completion';
 
-  constructor(private readonly configService: ConfigService) {
-    super();
+  constructor(
+    private readonly configService: ConfigService,
+    protected readonly httpService: HttpService,
+  ) {
+    super(httpService);
   }
 
   override async loadModel(model: Model): Promise<void> {
@@ -38,7 +44,7 @@ export default class CortexProvider extends OAIEngineExtension {
     // TODO: NamH check if the binary is there
 
     const cpuThreadCount = 1; // TODO: NamH Math.max(1, nitroResourceProbe.numCpuPhysicalCore);
-    const nitroModelSettings = {
+    const modelSettings = {
       // This is critical and requires real CPU physical core count (or performance core)
       cpu_threads: cpuThreadCount,
       ...model.settings,
@@ -55,38 +61,18 @@ export default class CortexProvider extends OAIEngineExtension {
       if (prompt?.error) {
         throw new Error(prompt.error);
       }
-      nitroModelSettings.system_prompt = prompt.system_prompt;
-      nitroModelSettings.user_prompt = prompt.user_prompt;
-      nitroModelSettings.ai_prompt = prompt.ai_prompt;
+      modelSettings.system_prompt = prompt.system_prompt;
+      modelSettings.user_prompt = prompt.user_prompt;
+      modelSettings.ai_prompt = prompt.ai_prompt;
     }
 
-    return fetch(LOAD_MODEL_URL, {
-      method: 'POST',
-      headers: {
-        'Content-Type': 'application/json',
-      },
-      body: JSON.stringify(nitroModelSettings),
-    })
-      .then((res) => {
-        if (!res.ok) {
-          throw new Error('Failed to load model');
-        }
-      })
-      .catch((e) => {
-        throw e;
-      });
+    await this.httpService.post(LOAD_MODEL_URL, modelSettings).toPromise();
   }
 
   override async unloadModel(modelId: string): Promise<void> {
-    return fetch(UNLOAD_MODEL_URL, {
-      method: 'POST',
-      headers: {
-        'Content-Type': 'application/json',
-      },
-      body: JSON.stringify({
-        model: modelId,
-      }),
-    }).then();
+    await this.httpService
+      .post(UNLOAD_MODEL_URL, { model: modelId })
+      .toPromise();
   }
 
   private readonly promptTemplateConverter = (

cortex-js/src/infrastructure/repositories/extensions/extension.module.ts

@@ -2,9 +2,10 @@ import { Module } from '@nestjs/common';
 import { ExtensionRepositoryImpl } from './extension.repository';
 import { ExtensionRepository } from '@/domain/repositories/extension.interface';
 import { CortexProviderModule } from '@/infrastructure/providers/cortex/cortex.module';
+import { HttpModule } from '@nestjs/axios';
 
 @Module({
-  imports: [CortexProviderModule],
+  imports: [CortexProviderModule, HttpModule],
   providers: [
     {
       provide: ExtensionRepository,

cortex-js/src/usecases/cortex/cortex.module.ts

@@ -1,8 +1,10 @@
 import { Module } from '@nestjs/common';
 import { CortexUsecases } from './cortex.usecases';
 import { CortexController } from '@/infrastructure/controllers/cortex.controller';
+import { HttpModule } from '@nestjs/axios';
 
 @Module({
+  imports: [HttpModule],
   providers: [CortexUsecases],
   controllers: [CortexController],
   exports: [CortexUsecases],

cortex-js/src/usecases/cortex/cortex.usecases.ts

@@ -4,12 +4,16 @@ import { ChildProcess, spawn } from 'child_process';
 import { join } from 'path';
 import { existsSync } from 'fs';
 import { CortexOperationSuccessfullyDto } from '@/infrastructure/dtos/cortex/cortex-operation-successfully.dto';
+import { HttpService } from '@nestjs/axios';
 
 @Injectable()
 export class CortexUsecases {
   private cortexProcess: ChildProcess | undefined;
 
-  constructor(private readonly configService: ConfigService) {}
+  constructor(
+    private readonly configService: ConfigService,
+    private readonly httpService: HttpService,
+  ) {}
 
   async startCortex(
     host: string,
@@ -70,25 +74,18 @@ export class CortexUsecases {
     host?: string,
     port?: number,
   ): Promise<CortexOperationSuccessfullyDto> {
-    if (this.cortexProcess) {
-      this.cortexProcess.kill();
+    try {
+      await this.httpService
+        .delete(`http://${host}:${port}/processmanager/destroy`)
+        .toPromise();
+    } catch (err) {
+      console.error(err);
+    } finally {
+      this.cortexProcess?.kill();
       return {
         message: 'Cortex stopped successfully',
         status: 'success',
       };
-    } else {
-      return new Promise((resolve) => {
-        return fetch(`http://${host}:${port}/processmanager/destroy`, {
-          method: 'DELETE',
-        })
-          .catch(() => {})
-          .finally(() => {
-            resolve({
-              message: 'Cortex stopped successfully',
-              status: 'success',
-            });
-          });
-      });
     }
   }
 

cortex-js/src/usecases/models/models.usecases.ts

@@ -108,7 +108,8 @@ export class ModelsUsecases {
           modelId: loadModelDto.modelId,
         };
       })
-      .catch(() => {
+      .catch((err) => {
+        console.error(err);
         return {
           message: 'Model failed to load',
           modelId: loadModelDto.modelId,