feat: add analytic metric, send benchmark to server

Author: marknguyen1302

cortex-js/src/app.ts

@@ -3,6 +3,7 @@ import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
 import { AppModule } from './app.module';
 import { FileManagerService } from './infrastructure/services/file-manager/file-manager.service';
 import { ValidationPipe } from '@nestjs/common';
+import { TelemetryUsecases } from './usecases/telemetry/telemetry.usecases';
 export const getApp = async () => {
   const app = await NestFactory.create(AppModule, {
     snapshot: true,
@@ -16,6 +17,9 @@ export const getApp = async () => {
   const fileService = app.get(FileManagerService);
   await fileService.getConfig();
 
+  const telemetryService = await app.resolve(TelemetryUsecases);
+  await telemetryService.initInterval();
+
   app.useGlobalPipes(
     new ValidationPipe({
       transform: true,

cortex-js/src/command.ts

@@ -15,21 +15,26 @@ async function bootstrap() {
     logger: ['warn', 'error'],
     errorHandler: async (error) => {
       await telemetryUseCase!.createCrashReport(error, TelemetrySource.CLI);
+      console.error(error);
       process.exit(1);
     },
     serviceErrorHandler: async (error) => {
       await telemetryUseCase!.createCrashReport(error, TelemetrySource.CLI);
+      console.error(error);
       process.exit(1);
     },
   });
 
   telemetryUseCase = await app.resolve(TelemetryUsecases);
   contextService = await app.resolve(ContextService);
 
-  telemetryUseCase!.sendCrashReport();
+  const anonymousData = await telemetryUseCase!.updateAnonymousData();
 
   await contextService!.init(async () => {
     contextService!.set('source', TelemetrySource.CLI);
+    contextService!.set('sessionId', anonymousData?.sessionId);
+    telemetryUseCase!.sendActivationEvent(TelemetrySource.CLI);
+    telemetryUseCase!.sendCrashReport();
     return CommandFactory.runApplication(app);
   });
 }

cortex-js/src/domain/repositories/telemetry.interface.ts

@@ -1,22 +1,50 @@
+import { ModelStat } from '@/infrastructure/commanders/types/model-stat.interface';
 import {
+  BenchmarkHardware,
   CrashReportAttributes,
+  EventAttributes,
   Telemetry,
+  TelemetryAnonymized,
   TelemetrySource,
 } from '../telemetry/telemetry.interface';
 
 export abstract class TelemetryRepository {
   abstract readCrashReports(
     callback: (Telemetry: Telemetry) => void,
   ): Promise<void>;
+
   abstract createCrashReport(
     crashReport: CrashReportAttributes,
     source?: TelemetrySource,
   ): Promise<void>;
+
   abstract getLastCrashReport(): Promise<Telemetry | null>;
+
   abstract markLastCrashReportAsSent(): Promise<void>;
+
   abstract sendTelemetryToOTelCollector(
     endpoint: string,
     telemetry: Telemetry,
   ): Promise<void>;
-  abstract sendTelemetryToServer(telemetry: Telemetry): Promise<void>;
+
+  abstract sendTelemetryToServer(
+    telemetryEvent: Telemetry['event'],
+  ): Promise<void>;
+
+  abstract sendEvent(
+    events: EventAttributes[],
+    source: TelemetrySource,
+  ): Promise<void>;
+
+  abstract getAnonymizedData(): Promise<TelemetryAnonymized | null>;
+
+  abstract updateAnonymousData(data: TelemetryAnonymized): Promise<void>;
+
+  abstract sendBenchmarkToServer(data: {
+    hardware: BenchmarkHardware;
+    results: any;
+    metrics: any;
+    model: ModelStat;
+    sessionId: string;
+  }): Promise<void>;
 }

cortex-js/src/domain/telemetry/crash-report.interface.ts

@@ -8,14 +8,7 @@ export interface TelemetryResource {
   source?: TelemetrySource;
   cpu?: string;
   gpus?: string;
-  // todo: consider about sessionId
-  // sessionId: string;
-}
-
-export interface CrashReportEvent {
-  modelId?: string;
-  sessionId: string;
-  stack?: string;
+  sessionId?: string;
 }
 
 export interface Resource {
@@ -77,6 +70,7 @@ export interface CrashReportAttributes {
   stack?: string;
   message: string;
   payload: CrashReportPayload;
+  sessionId?: string;
 }
 
 export interface TelemetryEventMetadata {
@@ -90,3 +84,31 @@ export interface Telemetry {
     resourceLogs: TelemetryEvent[];
   };
 }
+
+export enum EventName {
+  INIT = 'init',
+  DOWNLOAD_MODEL = 'download-model',
+  CHAT = 'chat',
+  ACTIVATE = 'activate',
+  NEW_ACTIVATE = 'new_activate',
+}
+
+export interface EventAttributes {
+  name: string;
+  modelId?: string;
+  sessionId?: string;
+}
+
+export interface TelemetryAnonymized {
+  sessionId: string | null;
+  lastActiveAt?: string | null;
+}
+export interface BenchmarkHardware {
+  gpu: any[];
+  cpu: any;
+  board: any;
+  disk: any[];
+  chassis: any;
+  memLayout: any[];
+  os: any;
+}

cortex-js/src/domain/telemetry/telemetry.interface.ts

@@ -10,6 +10,11 @@ import { PSCliUsecases } from './usecases/ps.cli.usecases';
 import { ModelsUsecases } from '@/usecases/models/models.usecases';
 import { SetCommandContext } from './decorators/CommandContext';
 import { ModelStat } from './types/model-stat.interface';
+import { TelemetryUsecases } from '@/usecases/telemetry/telemetry.usecases';
+import {
+  EventName,
+  TelemetrySource,
+} from '@/domain/telemetry/telemetry.interface';
 import { ContextService } from '../services/context/context.service';
 
 type ChatOptions = {
@@ -36,6 +41,7 @@ export class ChatCommand extends CommandRunner {
     private readonly modelsUsecases: ModelsUsecases,
     private readonly psCliUsecases: PSCliUsecases,
     readonly contextService: ContextService,
+    private readonly telemetryUsecases: TelemetryUsecases,
   ) {
     super();
   }
@@ -66,14 +72,23 @@ export class ChatCommand extends CommandRunner {
     }
 
     if (!message) options.attach = true;
-
-    return this.chatCliUsecases.chat(
+    const result = await this.chatCliUsecases.chat(
       modelId,
       options.threadId,
       message, // Accept both message from inputs or arguments
       options.attach,
       false, // Do not stop cortex session or loaded model
     );
+    this.telemetryUsecases.sendEvent(
+      [
+        {
+          name: EventName.CHAT,
+          modelId,
+        },
+      ],
+      TelemetrySource.CLI,
+    );
+    return result;
   }
 
   modelInquiry = async (models: ModelStat[]) => {

cortex-js/src/infrastructure/commanders/chat.command.ts

@@ -7,6 +7,11 @@ import {
 import { InitCliUsecases } from './usecases/init.cli.usecases';
 import { InitOptions } from './types/init-options.interface';
 import { SetCommandContext } from './decorators/CommandContext';
+import { TelemetryUsecases } from '@/usecases/telemetry/telemetry.usecases';
+import {
+  EventName,
+  TelemetrySource,
+} from '@/domain/telemetry/telemetry.interface';
 import { ContextService } from '../services/context/context.service';
 
 @SubCommand({
@@ -24,6 +29,7 @@ export class InitCommand extends CommandRunner {
     private readonly inquirerService: InquirerService,
     private readonly initUsecases: InitCliUsecases,
     readonly contextService: ContextService,
+    private readonly telemetryUsecases: TelemetryUsecases,
   ) {
     super();
   }
@@ -42,6 +48,14 @@ export class InitCommand extends CommandRunner {
       const version = passedParams[0] ?? 'latest';
 
       await this.initUsecases.installEngine(options, version);
+      this.telemetryUsecases.sendEvent(
+        [
+          {
+            name: EventName.INIT,
+          },
+        ],
+        TelemetrySource.CLI,
+      );
     }
   }
 

cortex-js/src/infrastructure/commanders/init.command.ts

@@ -3,6 +3,11 @@ import { exit } from 'node:process';
 import { SetCommandContext } from '../decorators/CommandContext';
 import { ModelsCliUsecases } from '@commanders/usecases/models.cli.usecases';
 import { ModelNotFoundException } from '@/infrastructure/exception/model-not-found.exception';
+import { TelemetryUsecases } from '@/usecases/telemetry/telemetry.usecases';
+import {
+  EventName,
+  TelemetrySource,
+} from '@/domain/telemetry/telemetry.interface';
 import { ContextService } from '@/infrastructure/services/context/context.service';
 import { existsSync } from 'fs';
 import { join } from 'node:path';
@@ -25,6 +30,7 @@ export class ModelPullCommand extends CommandRunner {
     private readonly initUsecases: InitCliUsecases,
     private readonly fileService: FileManagerService,
     readonly contextService: ContextService,
+    private readonly telemetryUsecases: TelemetryUsecases,
   ) {
     super();
   }
@@ -59,7 +65,15 @@ export class ModelPullCommand extends CommandRunner {
         engine,
       );
     }
-
+    this.telemetryUsecases.sendEvent(
+      [
+        {
+          name: EventName.DOWNLOAD_MODEL,
+          modelId: passedParams[0],
+        },
+      ],
+      TelemetrySource.CLI,
+    );
     console.log('\nDownload complete!');
     exit(0);
   }