Merge pull request #576 from janhq/chore/update-api

chore: remove inference settings

Author: namchuai

cortex-js/src/app.module.ts

@@ -6,28 +6,26 @@ import { DevtoolsModule } from '@nestjs/devtools-integration';
 import { DatabaseModule } from './infrastructure/database/database.module';
 import { ChatModule } from './usecases/chat/chat.module';
 import { AssistantsModule } from './usecases/assistants/assistants.module';
-import { InferenceSettingsModule } from './usecases/inference-settings/inference-settings.module';
 import { ExtensionModule } from './infrastructure/repositories/extensions/extension.module';
 import { CortexModule } from './usecases/cortex/cortex.module';
 import { ConfigModule } from '@nestjs/config';
+import { env } from 'node:process';
 
 @Module({
   imports: [
     DevtoolsModule.register({
-      http: process.env.NODE_ENV !== 'production',
+      http: env.NODE_ENV !== 'production',
     }),
     ConfigModule.forRoot({
       isGlobal: true,
-      envFilePath:
-        process.env.NODE_ENV === 'production' ? '.env' : '.env.development',
+      envFilePath: env.NODE_ENV !== 'production' ? '.env.development' : '.env',
     }),
     DatabaseModule,
     MessagesModule,
     ThreadsModule,
     ModelsModule,
     ChatModule,
     AssistantsModule,
-    InferenceSettingsModule,
     CortexModule,
     ExtensionModule,
   ],

cortex-js/src/infrastructure/commanders/inquirer/init.questions.ts

@@ -1,4 +1,5 @@
 import { Question, QuestionSet } from 'nest-commander';
+import { platform } from 'node:process';
 
 @QuestionSet({ name: 'create-init-questions' })
 export class CreateInitQuestions {
@@ -8,7 +9,7 @@ export class CreateInitQuestions {
     name: 'runMode',
     default: 'CPU',
     choices: ['CPU', 'GPU'],
-    when: () => process.platform !== 'darwin',
+    when: () => platform !== 'darwin',
   })
   parseRunMode(val: string) {
     return val;
@@ -31,7 +32,7 @@ export class CreateInitQuestions {
     message: 'Select CPU instructions set',
     name: 'instructions',
     choices: ['AVX2', 'AVX', 'AVX-512'],
-    when: () => process.platform !== 'darwin',
+    when: () => platform !== 'darwin',
   })
   parseContent(val: string) {
     return val;

cortex-js/src/infrastructure/commanders/pull.command.ts

@@ -34,9 +34,10 @@ export class PullCommand extends CommandRunner {
     super();
   }
 
-  async run(input: string[]): Promise<void> {
+  async run(input: string[]) {
     if (input.length < 1) {
-      return Promise.reject('Model ID is required');
+      console.error('Model ID is required');
+      exit(1);
     }
 
     const modelId = input[0];
@@ -46,7 +47,7 @@ export class PullCommand extends CommandRunner {
 
     const bar = new SingleBar({}, Presets.shades_classic);
     bar.start(100, 0);
-    await this.modelsUsecases.downloadModel({ modelId }, (progress) => {
+    await this.modelsUsecases.downloadModel(modelId, (progress) => {
       bar.update(progress);
     });
     console.log('\nDownload complete!');

cortex-js/src/infrastructure/commanders/start.command.ts

@@ -40,19 +40,7 @@ export class StartCommand extends CommandRunner {
   }
 
   private async startModel(modelId: string) {
-    // TODO: NamH remove these hardcoded value
-    const settings: ModelSettingParams = {
-      cpu_threads: 10,
-      ctx_len: 2048,
-      embedding: false,
-      prompt_template:
-        '{system_message}\n### Instruction: {prompt}\n### Response:',
-      system_prompt: '',
-      user_prompt: '\n### Instruction: ',
-      ai_prompt: '\n### Response:',
-      ngl: 100,
-    };
-    return this.modelsUsecases.startModel(modelId, settings);
+    return this.modelsUsecases.startModel(modelId);
   }
 
   private async getModelOrStop(modelId: string): Promise<Model> {

cortex-js/src/infrastructure/controllers/inference-settings.controller.spec.ts

@@ -13,55 +13,65 @@ import { CreateModelDto } from '@/infrastructure/dtos/models/create-model.dto';
 import { UpdateModelDto } from '@/infrastructure/dtos/models/update-model.dto';
 import { ApiResponse, ApiTags } from '@nestjs/swagger';
 import { StartModelSuccessDto } from '@/infrastructure/dtos/models/start-model-success.dto';
-import { DownloadModelDto } from '@/infrastructure/dtos/models/download-model.dto';
 import { ModelSettingParamsDto } from '../dtos/models/model-setting-params.dto';
 
 @ApiTags('Models')
 @Controller('models')
 export class ModelsController {
-  constructor(private readonly modelsService: ModelsUsecases) {}
+  constructor(private readonly modelsUsecases: ModelsUsecases) {}
 
   @Post()
   create(@Body() createModelDto: CreateModelDto) {
-    return this.modelsService.create(createModelDto);
+    return this.modelsUsecases.create(createModelDto);
   }
 
   @HttpCode(200)
   @ApiResponse({
     status: 200,
-    description: 'The model has been loaded successfully.',
+    description: 'The model has been started successfully.',
     type: StartModelSuccessDto,
   })
   @Post(':modelId/start')
   startModel(
     @Param('modelId') modelId: string,
     @Body() settings: ModelSettingParamsDto,
   ) {
-    return this.modelsService.startModel(modelId, settings);
+    return this.modelsUsecases.startModel(modelId, settings);
   }
 
-  @Post('download')
-  downloadModel(@Body() downloadModelDto: DownloadModelDto) {
-    return this.modelsService.downloadModel(downloadModelDto);
+  @HttpCode(200)
+  @ApiResponse({
+    status: 200,
+    description: 'The model has been stopped successfully.',
+    type: StartModelSuccessDto,
+  })
+  @Post(':modelId/stop')
+  stopModel(@Param('modelId') modelId: string) {
+    return this.modelsUsecases.stopModel(modelId);
+  }
+
+  @Get('download/:modelId')
+  downloadModel(@Param('modelId') modelId: string) {
+    return this.modelsUsecases.downloadModel(modelId);
   }
 
   @Get()
   findAll() {
-    return this.modelsService.findAll();
+    return this.modelsUsecases.findAll();
   }
 
   @Get(':id')
   findOne(@Param('id') id: string) {
-    return this.modelsService.findOne(id);
+    return this.modelsUsecases.findOne(id);
   }
 
   @Patch(':id')
   update(@Param('id') id: string, @Body() updateModelDto: UpdateModelDto) {
-    return this.modelsService.update(id, updateModelDto);
+    return this.modelsUsecases.update(id, updateModelDto);
   }
 
   @Delete(':id')
   remove(@Param('id') id: string) {
-    return this.modelsService.remove(id);
+    return this.modelsUsecases.remove(id);
   }
 }

cortex-js/src/infrastructure/controllers/inference-settings.controller.ts

@@ -4,7 +4,6 @@ import { sqliteDatabaseProviders } from './sqlite-database.providers';
 import { modelProviders } from './providers/model.providers';
 import { assistantProviders } from './providers/assistant.providers';
 import { messageProviders } from './providers/message.providers';
-import { inferenceSettingProviders } from './providers/inference-setting.providers';
 
 @Module({
   providers: [
@@ -13,14 +12,12 @@ import { inferenceSettingProviders } from './providers/inference-setting.provide
     ...modelProviders,
     ...assistantProviders,
     ...messageProviders,
-    ...inferenceSettingProviders,
   ],
   exports: [
     ...threadProviders,
     ...modelProviders,
     ...assistantProviders,
     ...messageProviders,
-    ...inferenceSettingProviders,
   ],
 })
 export class DatabaseModule {}

cortex-js/src/infrastructure/controllers/models.controller.ts

@@ -1,10 +1,22 @@
+import { ApiProperty } from '@nestjs/swagger';
 import { IsIP, IsNumber, IsString, Max, Min } from 'class-validator';
+import { defaultCortexCppHost, defaultCortexCppPort } from 'constant';
 
 export class StartCortexDto {
+  @ApiProperty({
+    name: 'host',
+    description: 'Cortexcpp host',
+    default: defaultCortexCppHost,
+  })
   @IsString()
   @IsIP()
   host: string;
 
+  @ApiProperty({
+    name: 'port',
+    description: 'Cortexcpp port',
+    default: defaultCortexCppPort,
+  })
   @IsNumber()
   @Min(0)
   @Max(65535)