Opensource models

packages/server/package.json

@@ -18,6 +18,7 @@
       "typescript": "^5"
    },
    "dependencies": {
+      "@huggingface/inference": "^4.13.0",
       "@prisma/client": "^6.16.2",
       "dayjs": "^1.11.18",
       "dotenv": "^17.2.2",

packages/server/prompts/summarizeReviewLlama.txt

@@ -0,0 +1,2 @@
+Summarize the following customer reviews into a short paragraph
+highlighting key themes, both positive and negative.

packages/server/providers/llm.provider.ts

@@ -1,4 +1,6 @@
 import OpenAI from 'openai';
+import { InferenceClient } from '@huggingface/inference';
+import summarizeReviewPrompt from '../prompts/summarizeReviewLlama.txt';
 
 type GenerateResponseQuery = {
    prompt: string;
@@ -12,6 +14,14 @@ type GenerateResponseResult = {
    id: string;
 };
 
+type SummarizeTextQuery = {
+   text: string;
+};
+
+type SummarizeTextResult = {
+   summary: string;
+};
+
 export class LlmProvider {
    constructor(private readonly openAiClient: OpenAI) {}
    async generateResponse({
@@ -31,10 +41,42 @@ export class LlmProvider {
 
       return { message: response.output_text, id: response.id };
    }
+
+   async summarize({ text }: SummarizeTextQuery): Promise<SummarizeTextResult> {
+      const output = await inferenceClient.summarization({
+         model: 'facebook/bart-large-cnn',
+         inputs: text,
+         provider: 'hf-inference',
+      });
+      return { summary: output.summary_text };
+   }
+
+   async summarizeReviews({
+      text: reviews,
+   }: SummarizeTextQuery): Promise<SummarizeTextResult> {
+      const chatCompletion = await inferenceClient.chatCompletion({
+         provider: 'novita',
+         model: 'meta-llama/Llama-3.1-8B-Instruct',
+         messages: [
+            {
+               role: 'system',
+               content: summarizeReviewPrompt,
+            },
+            {
+               role: 'user',
+               content: reviews,
+            },
+         ],
+      });
+
+      return { summary: chatCompletion?.choices[0]?.message.content ?? '' };
+   }
 }
 
 const openAiClient = new OpenAI({
    apiKey: process.env.OPENAI_API_KEY,
 });
 
+const inferenceClient = new InferenceClient(process.env.HF_TOKEN);
+
 export const llmProvider = new LlmProvider(openAiClient);

packages/server/services/review.service.ts

@@ -34,10 +34,15 @@ export class ReviewService {
       }
 
       const joinedReviews = reviews.map((r) => r.content).join('\n\n');
+      /*
+      Replace if you want to use OpenAI instead of an open-source model
       const prompt = template.replace('{{ reviews }}', joinedReviews);
       const { message: summary } = await this.llmProvider.generateResponse({
          prompt,
          maxOutputTokens: 500,
+      }); */
+      const { summary } = await this.llmProvider.summarizeReviews({
+         text: joinedReviews,
       });
       await this.reviewsRepository.upsertReviewSummary(productId, summary);
       return summary;