llm.txt generation (#94)

Author: lukasbach

package.json

@@ -15,6 +15,7 @@
     "build:web": "lerna run build:docs,build:sb --concurrency 1",
     "test": "lerna run test --stream",
     "postinstall": "zx scripts/prepare.mjs",
+    "llmtxt": "zx scripts/generate-llmtxt.mjs",
     "verify": "yarn lint && lerna run build,build:esm,test --stream"
   },
   "dependencies": {
@@ -24,6 +25,7 @@
     "@types/react": "18.2.66",
     "@types/react-dom": "18.2.22",
     "eslint": "^8.57.0",
+    "front-matter": "^4.0.2",
     "lerna": "^8.1.3",
     "typedoc": "^0.25.13",
     "typedoc-plugin-markdown": "^3.17.1",

packages/docs/.gitignore

@@ -21,3 +21,9 @@ yarn-error.log*
 
 docs/changelog/*.md
 docs/contributing/1-overview.mdx
+
+static/llm-full.txt
+static/llms-full.txt
+static/llm.txt
+static/llms.txt
+static/llm

scripts/generate-llmtxt.mjs

@@ -0,0 +1,57 @@
+#!/usr/bin/env zx
+
+import "zx/globals";
+import fm from "front-matter";
+
+const prefix = "https://headless-tree.lukasbach.com/llm";
+const { entries: sb } = await fs.readJSON(path.join(__dirname, `../packages/sb-react/storybook-static/index.json`));
+const files = await glob([`./packages/docs/docs/**/*.mdx`, `./packages/docs/docs/**/*.md`]);
+const slugs = [];
+
+let entrySmall = "# Headless Tree Documentation\n\n";
+let entryFull = `${entrySmall}`;
+
+const consumeMdx = async (file) => {
+    const content = await fs.readFile(file, { encoding: "utf8" });
+    const { attributes, body } = fm(content);
+    if (!attributes?.slug) return;
+
+    const bodyWithFixedLinks = body.replace(/\[(.*?)\]\((.*?)\)/g, (match, p1, p2) => {
+        if (p2.startsWith("http")) return match;
+        const matchedSlug = slugs.find(slug => p2 === slug || p2.startsWith(`${slug}#`));
+        if (matchedSlug) {
+            return `[${p1}](${prefix}${matchedSlug}.md)`;
+        }
+        return p1;
+    });
+
+    // fix <DemoBox stories={["abc"]} /> to ```abc```
+    const bodyWithFixedDemoBox = bodyWithFixedLinks.replace(/<DemoBox (.*?) \/>/g, (match, p1) => {
+        const story = /initialStory="([^"]*)"/.exec(p1)?.[1] ?? /stories=\{\["([^"]*)"/.exec(p1)?.[1];
+        const file = sb[story]?.importPath;
+        if (!file) return "";
+        const code = fs.readFileSync(path.join(__dirname, "../packages/sb-react", file), { encoding: "utf8" });
+        return `\`\`\`ts jsx\n${code}\`\`\``;
+    });
+
+    await fs.ensureDir(path.dirname(path.join(__dirname, "../packages/docs/static/llm", `${attributes.slug}.md`)));
+    await fs.writeFile(path.join(__dirname, "../packages/docs/static/llm", `${attributes.slug}.md`), bodyWithFixedDemoBox, { encoding: "utf8" });
+
+    entryFull += `\n---\n<!-- ${attributes.slug } -->\n# ${attributes.title}\n\n${bodyWithFixedDemoBox}\n`;
+    entrySmall += `\n- [${attributes.title}](${prefix}${attributes.slug}.md)${attributes.subtitle ? ": " : ""}${attributes.subtitle ?? ""}`;
+}
+
+for (const file of files) {
+    const content = await fs.readFile(file, { encoding: "utf8" });
+    const { attributes } = fm(content);
+    if (attributes?.slug) slugs.push(attributes.slug);
+}
+
+
+for (const file of files) {
+    await consumeMdx(file);
+}
+await fs.writeFile(path.join(__dirname, "../packages/docs/static/llms.txt"), entrySmall, { encoding: "utf8" });
+await fs.writeFile(path.join(__dirname, "../packages/docs/static/llm.txt"), entrySmall, { encoding: "utf8" });
+await fs.writeFile(path.join(__dirname, "../packages/docs/static/llm-full.txt"), entryFull, { encoding: "utf8" });
+await fs.writeFile(path.join(__dirname, "../packages/docs/static/llms-full.txt"), entryFull, { encoding: "utf8" });

yarn.lock

@@ -20591,6 +20591,7 @@ __metadata:
     "@types/react": "npm:18.2.66"
     "@types/react-dom": "npm:18.2.22"
     eslint: "npm:^8.57.0"
+    front-matter: "npm:^4.0.2"
     lerna: "npm:^8.1.3"
     typedoc: "npm:^0.25.13"
     typedoc-plugin-markdown: "npm:^3.17.1"