124 extract usefull information from the opening job

server/package.json

@@ -22,6 +22,8 @@
     "test:e2e": "jest --config ./test/jest-e2e.json"
   },
   "dependencies": {
+    "@anthropic-ai/sdk": "^0.78.0",
+    "@google/generative-ai": "^0.24.1",
     "@nestjs/axios": "^4.0.0",
     "@nestjs/cli": "^11.0.7",
     "@nestjs/common": "^11.1.0",
@@ -37,15 +39,21 @@
     "@types/bcrypt": "^5.0.2",
     "@types/cookie-parser": "^1.4.10",
     "@types/express": "^4.17.17",
-    "axios": "^1.9.0",
+    "axios": "^1.16.1",
     "bcrypt": "^6.0.0",
+    "cheerio": "^1.2.0",
     "class-transformer": "^0.5.1",
     "class-validator": "^0.14.1",
     "cookie-parser": "^1.4.7",
     "dotenv": "^16.5.0",
+    "groq-sdk": "^1.2.0",
     "ioredis": "^5.10.1",
+    "multer": "^2.1.1",
     "nodemailer": "^8.0.2",
+    "pdf-parse": "^2.4.5",
+    "pdf-parse-debugging-disabled": "^1.1.1",
     "pg": "^8.15.6",
+    "puppeteer": "^24.43.1",
     "reflect-metadata": "^0.2.0",
     "rxjs": "^7.8.1",
     "typeorm": "^0.3.22",
@@ -55,8 +63,10 @@
     "@nestjs/schematics": "^11.0.5",
     "@nestjs/testing": "^11.1.9",
     "@types/jest": "^29.5.14",
+    "@types/multer": "^2.1.0",
     "@types/node": "^20.3.1",
     "@types/nodemailer": "^6.4.17",
+    "@types/pdf-parse": "^1.1.5",
     "@types/supertest": "^6.0.0",
     "jest": "^29.5.0",
     "oxlint": "^1.11.2",

server/src/common/middleware/multer.config.ts

@@ -0,0 +1,25 @@
+import multer from "multer";
+import path from "path";
+
+// use memory storage to store the file in memory as a buffer
+const storage = multer.memoryStorage();
+
+export const upload = multer({
+  storage: storage,
+  limits: {
+    fileSize: 5 * 1024 * 1024, // Limite à 5 Mo
+  },
+  fileFilter: (req, file, cb) => {
+    // ckeck file type is pdf
+    const filetypes = /pdf/;
+    const mimetype = filetypes.test(file.mimetype);
+    const extname = filetypes.test(
+      path.extname(file.originalname).toLowerCase(),
+    );
+
+    if (mimetype && extname) {
+      return cb(null, true);
+    }
+    cb(new Error("Only PDF files are allowed!"));
+  },
+});

server/src/common/utils/JobOfferExtraction.ts

@@ -0,0 +1,204 @@
+import axios from "axios";
+import * as cheerio from "cheerio";
+
+export const scrapeLinkedin = async (url: string): Promise<string> => {
+  const maxRetries = 3;
+  let attempt = 0;
+  let pageText = "";
+
+  const userAgents = [
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36",
+  ];
+
+  while (attempt < maxRetries && !pageText) {
+    try {
+      // Délai croissant entre chaque tentative : 0ms, 2000ms, 4000ms
+      if (attempt > 0) {
+        const delay = attempt * 2000;
+        console.log(
+          `LinkedIn retry ${attempt}/${maxRetries - 1} - waiting ${delay}ms...`,
+        );
+        await new Promise((resolve) => setTimeout(resolve, delay));
+      }
+
+      const jobIdMatch = url.match(/(\d{8,})/);
+      if (!jobIdMatch)
+        throw new Error("Could not extract LinkedIn job ID from URL");
+
+      const jobId = jobIdMatch[1];
+      const guestApiUrl = `https://www.linkedin.com/jobs-guest/jobs/api/jobPosting/${jobId}`;
+      const response = await axios.get(guestApiUrl, {
+        headers: {
+          "User-Agent": userAgents[attempt % userAgents.length],
+          Accept:
+            "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+          "Accept-Language": "fr-FR,fr;q=0.9,en-US;q=0.8",
+          Referer: "https://www.linkedin.com/",
+        },
+        timeout: 15000,
+      });
+
+      const temp = cheerio.load(response.data);
+
+      const jobTitle = temp(
+        "h2.top-card-layout__title, h1.top-card-layout__title",
+      )
+        .text()
+        .trim();
+      const companyName = temp(
+        "a.topcard__org-name-link, span.topcard__org-name-link",
+      )
+        .text()
+        .trim();
+      const location = temp("span.topcard__flavor--bullet")
+        .first()
+        .text()
+        .trim();
+      const description = temp("div.show-more-less-html__markup")
+        .text()
+        .replace(/\s+/g, " ")
+        .trim();
+
+      const criteria: Record<string, string> = {};
+      temp("li.description__job-criteria-item").each((_: number, el: any) => {
+        const label = temp(el).find("h3").text().trim();
+        const value = temp(el).find("span").text().trim();
+        if (label && value) criteria[label] = value;
+      });
+
+      const extracted = `
+                Job Title: ${jobTitle}
+                Company: ${companyName}
+                Location: ${location}
+                Contract Type: ${criteria["Type de poste"] || criteria["Employment type"] || ""}
+                Seniority Level: ${criteria["Niveau hiérarchique"] || criteria["Seniority level"] || ""}
+                Industry: ${criteria["Secteur"] || criteria["Industries"] || ""}
+                Job Function: ${criteria["Fonction"] || criteria["Job function"] || ""}
+                Description: ${description}
+            `
+        .replace(/\s+/g, " ")
+        .trim();
+
+      if (extracted.length >= 100) {
+        pageText = extracted;
+        console.log(`Strategy LinkedIn succeeded on attempt ${attempt + 1}`);
+      } else {
+        throw new Error("Extracted content too short");
+      }
+    } catch (err) {
+      console.log(
+        `LinkedIn attempt ${attempt + 1} failed:`,
+        (err as any)?.code || err,
+      );
+      attempt++;
+    }
+  }
+  if (!pageText) {
+    console.log(
+      "All LinkedIn attempts failed, falling through to next strategy...",
+    );
+  }
+  return pageText;
+};
+
+// ─── AXIOS (sites simples) ───────────────────────────────────────────────────
+export const scrapeAxios = async (url: string): Promise<string> => {
+  let pageText = "";
+
+  try {
+    const response = await axios.get(url, {
+      headers: {
+        "User-Agent":
+          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+        Accept:
+          "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+        "Accept-Language": "fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7",
+        "Accept-Encoding": "gzip, deflate, br",
+        Connection: "keep-alive",
+        "Upgrade-Insecure-Requests": "1",
+        "Sec-Fetch-Dest": "document",
+        "Sec-Fetch-Mode": "navigate",
+        "Sec-Fetch-Site": "none",
+        "Cache-Control": "max-age=0",
+      },
+      timeout: 10000,
+      maxRedirects: 5,
+    });
+
+    const temp = cheerio.load(response.data);
+    temp(
+      "script, style, nav, footer, header, iframe, noscript, [aria-hidden='true']",
+    ).remove();
+    const extracted = temp("body").text().replace(/\s+/g, " ").trim();
+
+    if (extracted.length >= 300) {
+      pageText = extracted;
+      console.log("Strategy Axios succeeded");
+    }
+  } catch (err) {
+    console.log("Strategy Axios failed:", (err as any)?.code || err);
+  }
+
+  return pageText;
+};
+
+// ─── PUPPETEER GÉNÉRIQUE ─────────────────────────────────────────────────────
+export const scrapePuppeteer = async (url: string): Promise<string> => {
+  let pageText = "";
+
+  try {
+    const puppeteer = require("puppeteer");
+
+    const browser = await puppeteer.launch({
+      headless: true,
+      executablePath: "/usr/bin/google-chrome",
+      args: [
+        "--no-sandbox",
+        "--disable-setuid-sandbox",
+        "--disable-blink-features=AutomationControlled",
+        "--disable-infobars",
+        "--window-size=1920,1080",
+      ],
+    });
+
+    const page = await browser.newPage();
+
+    await page.evaluateOnNewDocument(() => {
+      Object.defineProperty(navigator, "webdriver", { get: () => false });
+      (window as any).chrome = { runtime: {} };
+    });
+
+    await page.setUserAgent(
+      "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    );
+
+    await page.setViewport({ width: 1920, height: 1080 });
+    await page.goto(url, { waitUntil: "networkidle2", timeout: 20000 });
+    await new Promise((resolve) => setTimeout(resolve, 2000));
+    await page.evaluate(() =>
+      window.scrollTo(0, document.body.scrollHeight / 2),
+    );
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+
+    const html = await page.content();
+    await browser.close();
+
+    const temp = cheerio.load(html);
+    temp("script, style, nav, footer, header, iframe, noscript").remove();
+    const extracted = temp("body").text().replace(/\s+/g, " ").trim();
+
+    if (extracted.length >= 300) {
+      pageText = extracted;
+      console.log("Strategy Puppeteer generic succeeded");
+    }
+  } catch (err) {
+    console.log(
+      "Strategy Puppeteer generic failed:",
+      (err as any)?.message || err,
+    );
+  }
+
+  return pageText;
+};

server/src/entities/userJobOffer.entity.ts

@@ -0,0 +1,85 @@
+import {
+  Entity,
+  Column,
+  PrimaryGeneratedColumn,
+  OneToOne,
+  JoinColumn,
+  UpdateDateColumn,
+  BeforeInsert,
+} from "typeorm";
+import { user } from "./user.entity";
+import { uuidv7 } from "uuidv7";
+
+@Entity()
+export class user_job_offer {
+  @PrimaryGeneratedColumn("uuid")
+  job_offer_id: string;
+
+  @Column({ nullable: false })
+  user_id: string;
+
+  // This part will establish a one-to-one relationship between the user_job_offer and user entities, allowing us to easily access the user associated with a given job offer.
+  // The onDelete: "CASCADE" option ensures that if a user is deleted, their associated job offer will also be automatically removed from the database.
+  @OneToOne(() => user, { onDelete: "CASCADE" })
+  @JoinColumn({ name: "user_id" })
+  user: user;
+
+  @Column({ nullable: true })
+  job_title: string;
+
+  @Column({ nullable: true })
+  company_name: string;
+
+  @Column({ type: "text", nullable: true })
+  company_description: string;
+
+  @Column({ nullable: true })
+  sector: string;
+
+  @Column({ nullable: true })
+  contract_type: string;
+
+  @Column({ nullable: true })
+  location: string;
+
+  @Column({ type: "simple-array", nullable: true })
+  required_skills: string[];
+
+  @Column({ type: "simple-array", nullable: true })
+  preferred_skills: string[];
+
+  @Column({ nullable: true })
+  required_experience: string;
+
+  @Column({ nullable: true })
+  required_education: string;
+
+  @Column({ type: "json", nullable: true })
+  missions: string[];
+
+  @Column({ type: "simple-array", nullable: true })
+  soft_skills: string[];
+
+  @Column({ type: "simple-array", nullable: true })
+  languages_required: string[];
+
+  @Column({ nullable: true })
+  salary_range: string;
+
+  @Column({ type: "simple-array", nullable: true })
+  company_values: string[];
+
+  @Column({ type: "text", nullable: true })
+  team_description: string;
+
+  @Column({ nullable: true })
+  offer_url: string;
+
+  @UpdateDateColumn()
+  updated_at: Date;
+
+  @BeforeInsert()
+  generateUUIDv7() {
+    if (!this.job_offer_id) this.job_offer_id = uuidv7();
+  }
+}