release 4/16

.cspell.json

@@ -21,12 +21,14 @@
     "pkgs",
     "psql",
     "qiita",
+    "replacen",
     "reqwest",
     "rustc",
     "safify",
     "stdenv",
     "supabase",
-    "swiper"
+    "swiper",
+    "zenki"
   ],
   "dictionaries": [
     "softwareTerms",
@@ -52,6 +54,7 @@
     "**/*.svg",
     "**/migration.sql",
     "**/data.json",
+    "**/server/src/seeds/json/**",
     "**/Cargo.*",
     "scraper/target",
     "**/rust-toolchain.toml",

.gitignore

@@ -3,6 +3,9 @@
 /.direnv
 /.husky
 
+.cache
+data.json
+
 # Logs
 logs
 *.log

biome.json

@@ -22,6 +22,7 @@
       "bun.lockb",
       "server/target",
       "data.json",
+      "server/src/seeds/json",
       "scraper/target",
       ".next",
       "next-env.d.ts",

flake.nix

@@ -32,7 +32,7 @@
       };
       unstable = nixpkgs-unstable.legacyPackages.${system};
 
-      rust-bin = pkgs.rust-bin.fromRustupToolchainFile ./scraper/rust-toolchain.toml;
+      rust-bin = pkgs.rust-bin.beta.latest.default; # pkgs.rust-bin.fromRustupToolchainFile ./scraper/rust-toolchain.toml;
       prisma = pkgs.callPackage ./server/prisma.nix {inherit prisma-utils;};
 
       common = {
@@ -62,8 +62,7 @@
       };
     in {
       packages.scraper = pkgs.callPackage ./scraper {toolchain = rust-bin;};
-      devShells.default = pkgs.mkShell common;
-      devShells.scraper = pkgs.mkShell {
+      devShells.default = pkgs.mkShell {
         inherit (common) env;
         packages =
           common.packages

scraper/sample.ts

@@ -0,0 +1,14 @@
+[
+  {
+    name: "zenki",
+    courses: [
+      {
+        name: "数理科学基礎",
+        teacher: "(人名)",
+        semester: "S1",
+        period: "月曜2限、水曜1限",
+        code: "30003 CAS-FC1871L1",
+      },
+    ],
+  },
+];

scraper/src/io.rs

@@ -1,6 +1,5 @@
 use crate::types::*;
 use anyhow::ensure;
-use sha2::{Digest, Sha256};
 use tokio::fs;
 use tokio::io::AsyncWriteExt;
 
@@ -10,13 +9,20 @@ pub async fn write_to(file: &mut fs::File, content: Entry) -> anyhow::Result<()>
     Ok(())
 }
 
-use crate::CACHE_DIR;
+use crate::cache_dir;
 
 pub async fn request(url: &str) -> anyhow::Result<String> {
     println!("[request] sending request to {}", url);
 
-    let hash = Sha256::digest(url.as_bytes());
-    let path = format!("{CACHE_DIR}/{:x}", hash);
+    let cache_key = url
+        .to_string()
+        .replacen("/", "_", 1000)
+        .replacen(":", "_", 1000)
+        .replacen("?", "_", 1000)
+        .replacen("&", "_", 1000)
+        .replacen("=", "_", 1000)
+        .to_string();
+    let path = format!("{}/{cache_key}", cache_dir());
     if let Ok(bytes) = fs::read(&path).await {
         if let Ok(text) = String::from_utf8(bytes) {
             return Ok(text);

scraper/src/main.rs

@@ -16,13 +16,16 @@ use scraper::{Html, Selector};
 use urls::URLS;
 
 const RESULT_FILE: &str = "./data.json";
-const CACHE_DIR: &str = "./.cache";
+
+fn cache_dir() -> String {
+    "./.cache".to_string()
+}
 
 #[tokio::main(flavor = "multi_thread")]
 async fn main() {
     println!("[log] starting...");
 
-    let _ = fs::DirBuilder::new().create(CACHE_DIR).await;
+    let _ = fs::DirBuilder::new().create(cache_dir()).await;
 
     let mut file = fs::File::create(RESULT_FILE)
         .await
@@ -59,7 +62,8 @@ async fn get_courses_of(base_url: &str) -> Vec<Course> {
     futures::future::join_all(courses)
         .await
         .into_iter()
-        .collect::<Vec<_>>()
+        .flatten()
+        .collect()
 }
 
 lazy_static! {

scraper/src/parser.rs

@@ -1,6 +1,6 @@
 use anyhow::anyhow;
 use lazy_static::lazy_static;
-use scraper::{Html, Selector};
+use scraper::{ElementRef, Html, Selector};
 
 use crate::types::*;
 
@@ -17,19 +17,24 @@ lazy_static! {
         Selector::parse(".catalog-page-detail-table-cell.code-cell").unwrap();
 }
 
-pub fn parse_course_info(html: Html) -> anyhow::Result<Course> {
-    Ok(Course {
-        name: select(&html, &NAME_SELECTOR, 1)?,
-        teacher: select(&html, &TEACHER_SELECTOR, 1)?,
-        semester: select_all(&html, &SEMESTER_SELECTOR, 1)?.join(","),
-        period: select(&html, &PERIOD_SELECTOR, 1)?,
-        code: select_all(&html, &CODE_SELECTOR, 1)?.join(" "),
-    })
+pub fn parse_course_info(html: Html) -> anyhow::Result<Vec<Course>> {
+    html.select(&Selector::parse(".catalog-page-detail-table-row").unwrap())
+        .skip(1)
+        .map(|el| {
+            Ok(Course {
+                name: select(&el, &NAME_SELECTOR)?,
+                teacher: select(&el, &TEACHER_SELECTOR)?,
+                semester: select_all(&el, &SEMESTER_SELECTOR)?.join(","),
+                period: select(&el, &PERIOD_SELECTOR)?,
+                code: select_all(&el, &CODE_SELECTOR)?.join(" "),
+            })
+        })
+        .collect()
 }
 
-fn select(html: &Html, selector: &Selector, nth: usize) -> anyhow::Result<String> {
-    html.select(selector)
-        .nth(nth)
+fn select(el: &ElementRef, selector: &Selector) -> anyhow::Result<String> {
+    el.select(selector)
+        .next()
         .ok_or(anyhow!(
             "Couldn't find matching element for selector {:?}",
             selector,
@@ -38,12 +43,12 @@ fn select(html: &Html, selector: &Selector, nth: usize) -> anyhow::Result<String
 }
 
 fn select_all<'a>(
-    html: &'a Html,
+    html: &'a ElementRef,
     selector: &'static Selector,
-    nth: usize,
+    // nth: usize,
 ) -> anyhow::Result<Vec<&'a str>> {
     html.select(selector)
-        .nth(nth)
+        .next()
         .ok_or(anyhow!(
             "Couldn't find matching element for selector {:?}",
             selector,

scraper/src/urls.rs

@@ -1,4 +1,8 @@
-pub static URLS: [(&str, &str); 10] = [
+pub static URLS: [(&str, &str); 11] = [
+    (
+      "zenki",
+      "https://catalog.he.u-tokyo.ac.jp/result?q=&type=all&faculty_id=&facet=%7B%22faculty_type%22%3A%5B%22jd%22%5D%7D&page=",
+    ),
     (
         "law",
         "https://catalog.he.u-tokyo.ac.jp/result?type=ug&faculty_id=1&page=",

server/src/seeds/insertKoukiCourses.ts

@@ -0,0 +1,106 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+
+import { prisma } from "../database/client";
+
+// 後期 (scraper) 形式のデータを読み込む。
+const FILE_PATH = path.join(__dirname, "data.json");
+
+// sample
+// [
+//   {
+//     name: "zenki",
+//     courses: [
+//       {
+//         name: "数理科学基礎",
+//         teacher: "(人名)",
+//         semester: "S1,S2",
+//         period: "月曜2限、水曜1限",
+//         code: "30003 CAS-FC1871L1",
+//       },
+//     ],
+//   },
+// ];
+
+async function main() {
+  const jsonData: {
+    courses: {
+      name: string;
+      teacher: string;
+      semester: string;
+      period: string;
+      code: string;
+    }[];
+  }[] = JSON.parse(fs.readFileSync(FILE_PATH, "utf-8"));
+  console.log(jsonData);
+
+  const coursesData = jsonData[0].courses
+    .filter((course) => course.semester.split("")[0] === "S")
+    .map((course) => {
+      const { code, name, teacher } = course;
+      return {
+        id: code.split(" ")[0],
+        name: name,
+        teacher: teacher,
+      };
+    });
+
+  await prisma.course.createMany({
+    data: coursesData,
+  });
+
+  const slotsData: {
+    day: "mon" | "tue" | "wed" | "thu" | "fri" | "sat" | "sun" | "other";
+    period: number;
+    courseId: string;
+  }[] = [];
+
+  for (const courseData of jsonData[0].courses) {
+    const { code, period } = courseData;
+
+    if (courseData.semester.split("")[0] !== "S") continue;
+
+    for (const p of period.split("、")) {
+      const [dayJp, periodStr] = p.split("曜");
+      const day =
+        dayJp === "月"
+          ? "mon"
+          : dayJp === "火"
+            ? "tue"
+            : dayJp === "水"
+              ? "wed"
+              : dayJp === "木"
+                ? "thu"
+                : dayJp === "金"
+                  ? "fri"
+                  : dayJp === "土"
+                    ? "sat"
+                    : dayJp === "日"
+                      ? "sun"
+                      : "other";
+
+      slotsData.push({
+        day,
+        period: Number.parseInt(periodStr?.split("")[0]) || 0,
+        courseId: code.split(" ")[0],
+      });
+    }
+  }
+
+  await prisma.slot.createMany({
+    data: slotsData,
+    skipDuplicates: true,
+  });
+
+  console.log("Data inserted successfully!");
+}
+
+main()
+  .then(async () => {
+    await prisma.$disconnect();
+  })
+  .catch(async (e) => {
+    console.error(e);
+    await prisma.$disconnect();
+    process.exit(1);
+  });