{post.title}
+-
+ {items.map((item, i) => (
+
- {item} + ))} +
{paragraph}
+ })} +diff --git a/.npmrc.example b/.npmrc.example index 55343a2..66ee9d0 100644 --- a/.npmrc.example +++ b/.npmrc.example @@ -1 +1 @@ -//registry.npmjs.org/:_authToken=YOUR_NPM_AUTOMATION_TOKEN_HERE \ No newline at end of file +//registry.npmjs.org/:_authToken=YOUR_NPM_AUTOMATION_TOKEN_HERE diff --git a/README.md b/README.md index efa1a84..3640485 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,32 @@ -# @llmstxt — `llms.txt`, `llms-full.txt`, and Markdown for agents +# @llmtxt — `llms.txt`, `llms-full.txt`, and Markdown for agents Generate `llms.txt` (table of contents) and `llms-full.txt` (full content) from your app’s pages, and optionally serve **any page as Markdown** via content negotiation (`Accept: text/markdown`). - Standard: [llmstxt.org](https://llmstxt.org) -- Works with: Next.js App Router (`@llmstxt/next`), any Node.js framework (`@llmstxt/core`), and Next.js middleware (`@llmstxt/middleware`) +- Works with: Next.js App Router (`@llmtxt/next`), any Node.js framework (`@llmtxt/core`), and Next.js middleware (`@llmtxt/middleware`) ## Packages | Package | What it does | |---|---| -| [`@llmstxt/core`](./packages/core) | Scan your `app/` directory and generate `llms.txt` + `llms-full.txt` | -| [`@llmstxt/next`](./packages/next) | Next.js App Router route handlers for `/llms.txt` and `/llms-full.txt` | -| [`@llmstxt/middleware`](./packages/middleware) | Next.js middleware: any page responds as Markdown on `Accept: text/markdown` | +| [`@llmtxt/core`](./packages/core) | Scan your `app/` directory and generate `llms.txt` + `llms-full.txt` | +| [`@llmtxt/next`](./packages/next) | Next.js App Router route handlers for `/llms.txt` and `/llms-full.txt` | +| [`@llmtxt/middleware`](./packages/middleware) | Next.js middleware: any page responds as Markdown on `Accept: text/markdown` | +| [`@llmtxt/react`](./packages/react) | Build-time helpers for React SPAs: generate `llms.txt` + `llms-full.txt` from a route list | ## Keywords (SEO) -llms.txt, llms-full.txt, llmstxt, markdown for agents, content negotiation, Next.js, middleware, App Router, AI, LLM, SEO, website indexing, documentation, crawlable content, sitemap alternative +llms.txt, llms-full.txt, llmtxt, markdown for agents, content negotiation, Next.js, middleware, App Router, AI, LLM, SEO, website indexing, documentation, crawlable content, sitemap alternative ## Table of Contents - [Quick Start — Next.js (App Router)](#quick-start--nextjs-app-router) - [Quick Start — Markdown for Agents (Next.js Middleware)](#quick-start--markdown-for-agents-nextjs-middleware) +- [Quick Start — React (Build-time, no backend)](#quick-start--react-build-time-no-backend) - [Quick Start — Other Frameworks](#quick-start--other-frameworks-express-hono-bun-etc) - [How It Works](#how-it-works) -- [API Reference (`@llmstxt/core`)](#api-reference-llmstxtcore) -- [Environment Variables (`@llmstxt/next`)](#environment-variables-llmstxtnext) +- [API Reference (`@llmtxt/core`)](#api-reference-llmtxtcore) +- [Environment Variables (`@llmtxt/next`)](#environment-variables-llmtxtnext) - [Tips for Better Descriptions](#tips-for-better-descriptions) - [Development (this repo)](#development-this-repo) - [License](#license) @@ -36,7 +38,7 @@ llms.txt, llms-full.txt, llmstxt, markdown for agents, content negotiation, Next ### Install ```bash -npm install @llmstxt/next +npm install @llmtxt/next ``` ### Add route handlers @@ -51,12 +53,12 @@ src/app/ **`src/app/llms.txt/route.ts`** (zero-config): ```ts -export { GET } from '@llmstxt/next' +export { GET } from '@llmtxt/next' ``` **`src/app/llms-full.txt/route.ts`** (zero-config): ```ts -import { createLlmsFullTxtHandler } from '@llmstxt/next' +import { createLlmsFullTxtHandler } from '@llmtxt/next' export const GET = createLlmsFullTxtHandler() ``` @@ -64,7 +66,7 @@ Set one of: - `NEXT_PUBLIC_APP_URL` (recommended) - `VERCEL_URL` (usually already set on Vercel) -> Tip: Set `NEXT_PUBLIC_APP_URL` explicitly in local development and production. `@llmstxt/next` will only fall back to `http://localhost:3000` when `NODE_ENV=development`. +> Tip: Set `NEXT_PUBLIC_APP_URL` explicitly in local development and production. `@llmtxt/next` will only fall back to `http://localhost:3000` when `NODE_ENV=development`. Visit: - `/llms.txt` @@ -74,7 +76,7 @@ Visit: ```ts // src/app/llms.txt/route.ts -import { createLlmsTxtHandler } from '@llmstxt/next' +import { createLlmsTxtHandler } from '@llmtxt/next' export const GET = createLlmsTxtHandler({ title: 'My SaaS App', @@ -85,7 +87,7 @@ export const GET = createLlmsTxtHandler({ ```ts // src/app/llms-full.txt/route.ts -import { createLlmsFullTxtHandler } from '@llmstxt/next' +import { createLlmsFullTxtHandler } from '@llmtxt/next' export const GET = createLlmsFullTxtHandler({ fetchTimeoutMs: 8000, @@ -105,14 +107,14 @@ Serve the Markdown version of **any page** when a client sends `Accept: text/mar ### Install ```bash -npm install @llmstxt/middleware +npm install @llmtxt/middleware ``` ### Add middleware ```ts // middleware.ts (project root) -export { middleware, config } from '@llmstxt/middleware' +export { middleware, config } from '@llmtxt/middleware' ``` This returns: @@ -131,14 +133,57 @@ For best Markdown quality in production, plug in `@mozilla/readability` + `turnd --- +## Quick Start — React (Build-time, no backend) + +For React SPAs (Vite/CRA/React Router), generate the files at build time and ship them from `public/`. + +```bash +npm install -D @llmtxt/react +``` + +Create a route list: + +```ts +// llmtxt.routes.ts +import type { LlmtxtRoute } from '@llmtxt/react' + +export const routes: LlmtxtRoute[] = [ + { path: '/', title: 'Home', description: 'What this site is about.' }, + { path: '/docs', title: 'Docs' }, +] +``` + +Generate: + +```ts +// scripts/generate-llms.ts +import path from 'path' +import { writeLlmsFiles } from '@llmtxt/react' +import { routes } from '../llmtxt.routes' + +await writeLlmsFiles({ + routes, + baseUrl: process.env.PUBLIC_SITE_URL!, // e.g. https://example.com + outDir: path.join(process.cwd(), 'public'), +}) +``` + +Run it after deploy (recommended) or against a preview server: + +```bash +PUBLIC_SITE_URL=https://example.com node scripts/generate-llms.ts +``` + +--- + ## Quick Start — Other Frameworks (Express, Hono, Bun, etc.) ```bash -npm install @llmstxt/core +npm install @llmtxt/core ``` ```ts -import { generateLlmsTxt, generateLlmsFullTxt } from '@llmstxt/core' +import { generateLlmsTxt, generateLlmsFullTxt } from '@llmtxt/core' import path from 'path' app.get('/llms.txt', async (req, res) => { @@ -167,7 +212,7 @@ app.get('/llms-full.txt', async (req, res) => { ## How It Works ### `llms.txt` (Table of contents) -`@llmstxt/core` scans your `app/` directory for `page.tsx`/`page.jsx`/`page.ts`/`page.js` files and produces a structured +`@llmtxt/core` scans your `app/` directory for `page.tsx`/`page.jsx`/`page.ts`/`page.js` files and produces a structured index of links + descriptions. LLMs use this like a sitemap to understand what your site covers. Example output: @@ -183,14 +228,14 @@ Example output: - [Advanced Usage](https://example.com/blog/advanced): Deep-dives and recipes --- -*Generated by @llmstxt/core · 2025-01-01T00:00:00.000Z · 12 pages* +*Generated by @llmtxt/core · 2025-01-01T00:00:00.000Z · 12 pages* ``` ### `llms-full.txt` (Full content) Same scan, but also fetches and converts each page to Markdown (or plain text by default). This produces one big file containing your site’s content for ingestion. ### `Accept: text/markdown` (Markdown for agents) -`@llmstxt/middleware` adds content negotiation to your Next.js app: when a client requests any URL with `Accept: text/markdown`, it re-fetches that page as HTML, converts it to Markdown, and returns it with `Content-Type: text/markdown`, `Vary: Accept`, and `x-markdown-tokens`. +`@llmtxt/middleware` adds content negotiation to your Next.js app: when a client requests any URL with `Accept: text/markdown`, it re-fetches that page as HTML, converts it to Markdown, and returns it with `Content-Type: text/markdown`, `Vary: Accept`, and `x-markdown-tokens`. ## Why this helps SEO (and AI discoverability) @@ -199,7 +244,7 @@ These files are primarily for **LLM discoverability**: they provide a structured --- -## API Reference (`@llmstxt/core`) +## API Reference (`@llmtxt/core`) ### `generateLlmsTxt(options)` Generates `llms.txt` as a string. @@ -235,14 +280,14 @@ Generates `llms-full.txt` as a string (fetches pages and converts HTML→text/Ma --- -## Environment Variables (`@llmstxt/next`) +## Environment Variables (`@llmtxt/next`) | Variable | Purpose | |---|---| | `NEXT_PUBLIC_APP_URL` | Your public app root URL, e.g. `https://example.com` — recommended for production and local dev | | `VERCEL_URL` | Automatically set by Vercel; used as a fallback if `NEXT_PUBLIC_APP_URL` is missing | -> Note: When `NODE_ENV=development`, `@llmstxt/next` will default to `http://localhost:3000` if neither variable is set. For predictable results, always set `NEXT_PUBLIC_APP_URL` in `.env.local` or your deployment environment. +> Note: When `NODE_ENV=development`, `@llmtxt/next` will default to `http://localhost:3000` if neither variable is set. For predictable results, always set `NEXT_PUBLIC_APP_URL` in `.env.local` or your deployment environment. --- @@ -250,16 +295,16 @@ Generates `llms-full.txt` as a string (fetches pages and converts HTML→text/Ma This repository publishes three npm packages: -- `@llmstxt/core` — framework-agnostic generator for `llms.txt` and `llms-full.txt` -- `@llmstxt/next` — Next.js App Router route handlers for `llms.txt` and `llms-full.txt` -- `@llmstxt/middleware` — Next.js middleware that serves Markdown on `Accept: text/markdown` +- `@llmtxt/core` — framework-agnostic generator for `llms.txt` and `llms-full.txt` +- `@llmtxt/next` — Next.js App Router route handlers for `llms.txt` and `llms-full.txt` +- `@llmtxt/middleware` — Next.js middleware that serves Markdown on `Accept: text/markdown` Install only the packages you need: ```bash -npm install @llmstxt/core -npm install @llmstxt/next -npm install @llmstxt/middleware +npm install @llmtxt/core +npm install @llmtxt/next +npm install @llmtxt/middleware ``` See each package README for the full option reference. diff --git a/examples/README.md b/examples/README.md index 9902893..427dd05 100644 --- a/examples/README.md +++ b/examples/README.md @@ -2,8 +2,8 @@ This folder contains two example projects for local testing of the monorepo packages: -- `basic-test` — a simple TypeScript script that imports `@llmstxt/core` and generates `llms.txt` locally. -- `next-app-test` — a minimal Next.js App Router project that consumes `@llmstxt/next` and exposes `/llms.txt` and `/llms-full.txt`. +- `basic-test` — a simple TypeScript script that imports `@llmtxt/core` and generates `llms.txt` locally. +- `next-app-test` — a minimal Next.js App Router project that consumes `@llmtxt/next` and exposes `/llms.txt` and `/llms-full.txt`. ## Getting started diff --git a/examples/my-react-app/.gitignore b/examples/my-react-app/.gitignore new file mode 100644 index 0000000..703f5d2 --- /dev/null +++ b/examples/my-react-app/.gitignore @@ -0,0 +1,28 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-ssr +*.local + +# Generated llms files (generated by scripts/generate-llms.js) +public/llms.txt +public/llms-full.txt + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? diff --git a/examples/my-react-app/README.md b/examples/my-react-app/README.md new file mode 100644 index 0000000..48de779 --- /dev/null +++ b/examples/my-react-app/README.md @@ -0,0 +1,145 @@ +# React Demo App with @llmtxt/react + +A comprehensive example React application demonstrating how to use `@llmtxt/react` to generate AI-friendly documentation from your Vite + React SPA. + +## Features + +- **11+ Demo Pages** including Home, Docs, Pricing, Blog, and more +- **React Router** for full SPA navigation +- **Responsive Design** with modern CSS styling +- **@llmtxt/react Integration** for automatic documentation generation + +## Pages Included + +### Core Pages + +- **Home** (`/`) - Welcome and feature overview +- **Documentation** (`/docs`) - Documentation index + - Getting Started (`/docs/getting-started`) + - API Reference (`/docs/api`) +- **Features** (`/features`) - Feature showcase +- **Pricing** (`/pricing`) - Pricing plans +- **Blog** (`/blog`) - Blog post listing + - Individual Posts (`/blog/:slug`) +- **About** (`/about`) - About page +- **Contact** (`/contact`) - Contact form +- **FAQ** (`/faq`) - Frequently asked questions + +## Setup + +### 1. Install Dependencies + +```bash +npm install +``` + +### 2. Development Server + +```bash +npm run dev +``` + +Then visit `http://localhost:5173` + +### 3. Generate Documentation + +After deploying your app, generate the documentation: + +```bash +PUBLIC_SITE_URL=http://localhost:5173 npm run generate:llms +``` + +> Note: Because this is a client-rendered SPA, plain fetch() returns `index.html` for every route. +> For “real” `llms-full.txt` content, use prerender/SSR HTML, provide `route.markdown`, or render with Playwright: +> +> ```bash +> npm i -D playwright +> LLMTXT_RENDER=playwright PUBLIC_SITE_URL=http://localhost:5173 npm run generate:llms +> ``` + +This creates: + +- `public/llms.txt` - Lightweight documentation index +- `public/llms-full.txt` - Full documentation with page content + +## How It Works + +1. **Route Discovery**: + - Static routes are auto-scanned from `src/pages/**` via `scanPagesDirForRoutes()` + - Dynamic routes (e.g. `/blog/:slug`) must be expanded into concrete paths in the generator script + +2. **Generation Script** (`scripts/generate-llms.js`): + - Fetches each route from your running app + - Converts HTML to Markdown + - Writes documentation files to `public/` + +3. **Build Process**: + - `npm run build` runs Vite build + generation script + - Files are included in your deployment + +## File Structure + +``` +my-react-app/ +├── src/ +│ ├── pages/ # Individual page components +│ ├── App.jsx # Main app with routing +│ ├── App.css # Styling +│ └── main.jsx # Entry point +├── scripts/ +│ └── generate-llms.js # Documentation generator +├── src/pages/ # Convention-based pages (used for auto-scan) +└── package.json # Dependencies +``` + +## Customization + +### Add a New Page + +1. Create a component in `src/pages/MyPage.jsx` +2. Add route to `App.jsx` +3. Add/update routes in `scripts/generate-llms.js` + +### Customize Documentation Generation + +Edit `scripts/generate-llms.js`: + +```js +await writeLlmsFiles({ + routes, + baseUrl: process.env.PUBLIC_SITE_URL || 'http://localhost:5173', + outDir: path.join(process.cwd(), 'public'), + fetchTimeoutMs: 5000, // Per-page timeout + // htmlToMarkdown: customConverter // Optional custom converter +}) +``` + +## Viewing Generated Documentation + +After running `npm run generate:llms`: + +```bash +cat public/llms.txt # Lightweight index +cat public/llms-full.txt # Full documentation +``` + +## Environment Variables + +- `PUBLIC_SITE_URL` - Your app's public URL (used for documentation generation) + +For local development: + +```bash +PUBLIC_SITE_URL=http://localhost:5173 +``` + +For production: + +```bash +PUBLIC_SITE_URL=https://your-app.com +``` + +## Learn More + +- [@llmtxt/react](../../packages/react/) - Package documentation +- [llmstxt.org](https://llmstxt.org) - Format specification diff --git a/examples/my-react-app/eslint.config.js b/examples/my-react-app/eslint.config.js new file mode 100644 index 0000000..ea36dd3 --- /dev/null +++ b/examples/my-react-app/eslint.config.js @@ -0,0 +1,21 @@ +import js from '@eslint/js' +import globals from 'globals' +import reactHooks from 'eslint-plugin-react-hooks' +import reactRefresh from 'eslint-plugin-react-refresh' +import { defineConfig, globalIgnores } from 'eslint/config' + +export default defineConfig([ + globalIgnores(['dist']), + { + files: ['**/*.{js,jsx}'], + extends: [ + js.configs.recommended, + reactHooks.configs.flat.recommended, + reactRefresh.configs.vite, + ], + languageOptions: { + globals: globals.browser, + parserOptions: { ecmaFeatures: { jsx: true } }, + }, + }, +]) diff --git a/examples/my-react-app/index.html b/examples/my-react-app/index.html new file mode 100644 index 0000000..7b228d7 --- /dev/null +++ b/examples/my-react-app/index.html @@ -0,0 +1,13 @@ + + +
+ + + +
+ This demo app exists to show how to generate llms.txt and{' '}
+ llms-full.txt for a React SPA using @llmtxt/react.
+
{paragraph}
+ })} +Latest news, tips, and updates
+ +{post.excerpt}
+ + Read More → + +Email: hello@example.com
+Generates llms.txt and llms-full.txt files.
+ +routes (required): routes to include
+ baseUrl (required): public site URL
+ outDir (required): usually public
+ title/summary: llms.txt header content
+ fetchHtml: render SPAs with Playwright/Puppeteer instead of plain fetch
+ route.markdown: precomputed Markdown per route (best for dynamic routes)
+
+ Convention-based route discovery for projects that keep page components in{' '}
+ src/pages/**. Dynamic files like [slug].tsx are skipped by default.
+
Install the package as a dev dependency:
+npm install -D @llmtxt/react+
+ You can either provide an explicit route list, or (if you follow a pages-folder + convention) auto-scan it. +
+{`import path from 'path'
+import { scanPagesDirForRoutes } from '@llmtxt/react'
+
+const routes = await scanPagesDirForRoutes({
+ pagesDir: path.join(process.cwd(), 'src/pages'),
+ skipDynamic: true,
+})`}
+
+ Dynamic routes like /blog/:slug must be expanded into concrete paths (e.g.{' '}
+ /blog/introducing-v2) in your generator script.
+
Create scripts/generate-llms.js:
{`import path from 'path'
+import { scanPagesDirForRoutes, writeLlmsFiles } from '@llmtxt/react'
+
+const baseUrl = process.env.PUBLIC_SITE_URL || 'http://localhost:5173'
+const outDir = path.join(process.cwd(), 'public')
+const pagesDir = path.join(process.cwd(), 'src/pages')
+
+const routes = await scanPagesDirForRoutes({ pagesDir, skipDynamic: true })
+
+await writeLlmsFiles({
+ routes: [
+ ...routes,
+ // expand dynamic routes here:
+ { path: '/blog/introducing-v2', title: 'Introducing v2' },
+ ],
+ baseUrl,
+ outDir,
+})`}
+ After deploying your app, run:
+PUBLIC_SITE_URL=https://example.com npm run generate:llms+
This will fetch your pages and generate llms.txt and llms-full.txt files.
+Complete documentation and resources for using @llmtxt/react
+ +Quick start guide to set up @llmtxt/react in your project
+ +Complete API documentation with examples and use cases
+ +Auto-scan `src/pages/**` or provide an explicit route list
+Expand `/blog/:slug` into real paths like `/blog/introducing-v2`
+route.markdown, or Playwright rendering.
+ /blog/:slug into concrete
+ paths like /blog/introducing-v2.
+ Powerful capabilities for documentation generation
+ +{feature.description}
+A comprehensive example showcasing @llmtxt/react integration
+Complete guides and API reference to get you started quickly
+Optimized for speed with minimal bundle size
+Simple setup process that works with your existing React app
+Works perfectly on desktop, tablet, and mobile devices
+Check out our documentation to learn more about @llmtxt/react
+ + Read Documentation + +Simple, transparent pricing for everyone
+ +Complete API documentation for all llmstxt packages
+Complete API documentation for all llmtxt packages
- We're excited to announce llmstxt, a framework-agnostic solution for automatically generating + We're excited to announce llmtxt, a framework-agnostic solution for automatically generating AI-friendly documentation from your Next.js applications.
- Install the @llmstxt packages using npm: + Install the @llmtxt packages using npm:
-